{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d4ef440a",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Chapter 2: Launching an Application with Proprietary Models \n",
    "    Overview of Proprietary Models\n",
    "    Introduction to OpenAI + Embeddings / GPT3 / ChatGPT\n",
    "    Introduction to Vector Databases\n",
    "    Building a Neural/Semantic Information Retrieval System with Vector Databases, BERT & GPT3\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "4a469ed4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1536"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import openai\n",
    "from openai.embeddings_utils import get_embeddings, get_embedding\n",
    "\n",
    "openai.api_key = os.environ.get('OPENAI_API_KEY')\n",
    "\n",
    "ENGINE = 'text-embedding-ada-002'\n",
    "\n",
    "embedded_text = get_embedding('I love to be vectorized', engine=ENGINE)\n",
    "\n",
    "len(embedded_text) == '1536'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "c12182ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "from openai.embeddings_utils import get_embeddings, get_embedding\n",
    "from datetime import datetime\n",
    "import hashlib\n",
    "import re\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "\n",
    "import logging\n",
    "\n",
    "logger = logging.getLogger()\n",
    "logger.setLevel(logging.DEBUG)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "bb58aa1e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: OPENAI_API_KEY=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\n",
      "env: PINECONE_API_KEY=97aab1fa-afee-4e7a-9a2b-91965192d042\n"
     ]
    }
   ],
   "source": [
    "%env OPENAI_API_KEY=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\n",
    "%env PINECONE_API_KEY=97aab1fa-afee-4e7a-9a2b-91965192d042\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f0289c54",
   "metadata": {},
   "outputs": [],
   "source": [
    "openai.api_key = os.environ.get('OPENAI_API_KEY')\n",
    "pinecone_key = os.environ.get('PINECONE_API_KEY')\n",
    "\n",
    "INDEX_NAME = 'semantic-search'\n",
    "NAMESPACE = 'default'\n",
    "ENGINE = 'text-embedding-ada-002'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "544d9d2f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b9befee1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pinecone\n",
    "\n",
    "pinecone.init(api_key=pinecone_key, environment=\"us-west1-gcp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fac98396",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ea70672a",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not INDEX_NAME in pinecone.list_indexes():\n",
    "    pinecone.create_index(\n",
    "        INDEX_NAME,  # The name of the index\n",
    "        dimension=1536,  # The dimensionality of the vectors\n",
    "        metric='cosine',  # The similarity metric to use when searching the index\n",
    "        pod_type=\"p1\"  # The type of Pinecone pod\n",
    "    )\n",
    "\n",
    "# Store the index as a variable\n",
    "index = pinecone.Index(INDEX_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8088892",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6103d6c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7f2fdfe7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ae76cc4dfd345ecaeea9b8ba0d5c3437'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def my_hash(s):\n",
    "    # Return the MD5 hash of the input string as a hexadecimal string\n",
    "    return hashlib.md5(s.encode()).hexdigest()\n",
    "\n",
    "my_hash('I love to hash it')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ecd86f51",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_for_pinecone(texts, engine=ENGINE):\n",
    "    # Get the current UTC date and time\n",
    "    now = datetime.utcnow()\n",
    "    \n",
    "    # Generate vector embeddings for each string in the input list, using the specified engine\n",
    "    embeddings = get_embeddings(texts, engine=engine)\n",
    "    \n",
    "    # Create tuples of (hash, embedding, metadata) for each input string and its corresponding vector embedding\n",
    "    # The my_hash() function is used to generate a unique hash for each string, and the datetime.utcnow() function is used to generate the current UTC date and time\n",
    "    return [\n",
    "        (\n",
    "            my_hash(text),  # A unique ID for each string, generated using the my_hash() function\n",
    "            embedding,  # The vector embedding of the string\n",
    "            dict(text=text, date_uploaded=now)  # A dictionary of metadata, including the original text and the current UTC date and time\n",
    "        ) \n",
    "        for text, embedding in zip(texts, embeddings)  # Iterate over each input string and its corresponding vector embedding\n",
    "    ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4c40d99a",
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = ['hi']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e065c037",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('49f68a5c8493ec2c0bf489821c21fc3b',\n",
       " [-0.035126980394124985,\n",
       "  -0.020624293014407158,\n",
       "  -0.015343423001468182,\n",
       "  -0.03980357199907303,\n",
       "  -0.02750781551003456,\n",
       "  0.02111034281551838,\n",
       "  -0.022069307044148445,\n",
       "  -0.019442008808255196,\n",
       "  -0.00955679826438427,\n",
       "  -0.013143060728907585,\n",
       "  0.029583381488919258,\n",
       "  -0.004725852981209755,\n",
       "  -0.015198921784758568,\n",
       "  -0.014069183729588985,\n",
       "  0.00897879246622324,\n",
       "  0.01521205808967352,\n",
       "  0.03838483244180679,\n",
       "  -0.005753783974796534,\n",
       "  0.02394782565534115,\n",
       "  -0.012794943526387215,\n",
       "  -0.014936191961169243,\n",
       "  -0.0030887178145349026,\n",
       "  -0.006890090182423592,\n",
       "  -0.008466469123959541,\n",
       "  -0.022726131603121758,\n",
       "  -0.0001311596715822816,\n",
       "  0.013464905321598053,\n",
       "  -0.01697234809398651,\n",
       "  0.0044926805421710014,\n",
       "  -0.02233203686773777,\n",
       "  0.014528960920870304,\n",
       "  -0.0009466484771110117,\n",
       "  -0.04495307803153992,\n",
       "  -0.00971443671733141,\n",
       "  -0.00978011917322874,\n",
       "  -0.015724381431937218,\n",
       "  0.00985236931592226,\n",
       "  -0.02121543511748314,\n",
       "  0.015093830414116383,\n",
       "  -0.005523894913494587,\n",
       "  0.008295695297420025,\n",
       "  -0.005077254492789507,\n",
       "  0.0069426363334059715,\n",
       "  -0.013307266868650913,\n",
       "  -0.01827286183834076,\n",
       "  -0.008387650363147259,\n",
       "  -0.0022873918060213327,\n",
       "  0.006390903610736132,\n",
       "  -0.010141372680664062,\n",
       "  0.02086075022816658,\n",
       "  0.021754032000899315,\n",
       "  0.005208619404584169,\n",
       "  -0.028085820376873016,\n",
       "  -0.010916424915194511,\n",
       "  -0.016368070617318153,\n",
       "  -0.006683190818876028,\n",
       "  -0.014305640012025833,\n",
       "  0.01438445970416069,\n",
       "  -0.011231700889766216,\n",
       "  -0.018574999645352364,\n",
       "  -0.01014794036746025,\n",
       "  0.004006630275398493,\n",
       "  -0.0026782024651765823,\n",
       "  0.01218409650027752,\n",
       "  0.006101900711655617,\n",
       "  0.001293944544158876,\n",
       "  0.02233203686773777,\n",
       "  -0.0003121969639323652,\n",
       "  0.0008924604626372457,\n",
       "  0.00682440772652626,\n",
       "  0.030161386355757713,\n",
       "  0.03003002144396305,\n",
       "  -0.004584636073559523,\n",
       "  -0.013530587777495384,\n",
       "  0.024249965324997902,\n",
       "  -0.011606091633439064,\n",
       "  -0.00031794418464414775,\n",
       "  0.004387588705867529,\n",
       "  -0.006758725270628929,\n",
       "  -0.008190602995455265,\n",
       "  0.026417486369609833,\n",
       "  -0.02664080634713173,\n",
       "  -0.014844236895442009,\n",
       "  0.01564556173980236,\n",
       "  0.002589531010016799,\n",
       "  0.014502687379717827,\n",
       "  0.0004934394964948297,\n",
       "  0.01995433308184147,\n",
       "  -0.012446826323866844,\n",
       "  -0.03200049698352814,\n",
       "  0.006289095617830753,\n",
       "  0.025721251964569092,\n",
       "  0.014029773883521557,\n",
       "  0.01025303266942501,\n",
       "  -0.011402475647628307,\n",
       "  0.020913295447826385,\n",
       "  -0.0030887178145349026,\n",
       "  0.013221879489719868,\n",
       "  -0.008991928771138191,\n",
       "  0.0052184718661010265,\n",
       "  0.0015911576338112354,\n",
       "  0.0080329654738307,\n",
       "  -0.007218502461910248,\n",
       "  -0.006200424395501614,\n",
       "  -0.0368347242474556,\n",
       "  -0.015593016520142555,\n",
       "  -0.014620915986597538,\n",
       "  -0.007717689499258995,\n",
       "  0.017090577632188797,\n",
       "  -0.00955679826438427,\n",
       "  -0.006962341256439686,\n",
       "  0.03686099871993065,\n",
       "  -0.00252056447789073,\n",
       "  -0.032289497554302216,\n",
       "  -0.010706241242587566,\n",
       "  -0.007100274320691824,\n",
       "  0.009832664392888546,\n",
       "  2.6426929252920672e-05,\n",
       "  -0.019520828500390053,\n",
       "  -0.02243712916970253,\n",
       "  0.006896658800542355,\n",
       "  -0.004768546670675278,\n",
       "  0.00878831371665001,\n",
       "  0.0019031494157388806,\n",
       "  0.017616037279367447,\n",
       "  0.008078943006694317,\n",
       "  -0.029583381488919258,\n",
       "  -0.007047728169709444,\n",
       "  0.008111784234642982,\n",
       "  0.0017126702005043626,\n",
       "  0.04308769479393959,\n",
       "  0.012335166335105896,\n",
       "  0.006239833775907755,\n",
       "  -0.008046101778745651,\n",
       "  -0.0326835922896862,\n",
       "  0.030003748834133148,\n",
       "  -0.024315647780895233,\n",
       "  0.027665453031659126,\n",
       "  -0.021793441846966743,\n",
       "  -0.03305141627788544,\n",
       "  -0.010594581253826618,\n",
       "  0.03714999929070473,\n",
       "  -0.0023202330339699984,\n",
       "  -0.021333664655685425,\n",
       "  -0.01379331760108471,\n",
       "  -0.010745651088654995,\n",
       "  0.017721127718687057,\n",
       "  -0.002599383471533656,\n",
       "  -0.0038292875979095697,\n",
       "  -0.002241414040327072,\n",
       "  0.0028046411462128162,\n",
       "  -0.0008275990257970989,\n",
       "  0.022831223905086517,\n",
       "  -0.0015353275230154395,\n",
       "  0.028716372326016426,\n",
       "  0.02260790392756462,\n",
       "  -0.007941009476780891,\n",
       "  -0.010614286176860332,\n",
       "  -0.009491115808486938,\n",
       "  -0.0062792436219751835,\n",
       "  0.0008743978105485439,\n",
       "  -0.004315337631851435,\n",
       "  0.02313336357474327,\n",
       "  -0.0063547780737280846,\n",
       "  -0.005073970183730125,\n",
       "  0.020243335515260696,\n",
       "  0.020808205008506775,\n",
       "  0.029557108879089355,\n",
       "  0.011895094066858292,\n",
       "  -0.0006933605181984603,\n",
       "  0.010798197239637375,\n",
       "  0.022095579653978348,\n",
       "  -0.0305554810911417,\n",
       "  0.005021424498409033,\n",
       "  -0.0033481635618954897,\n",
       "  0.014923055656254292,\n",
       "  0.010417238809168339,\n",
       "  0.01585574634373188,\n",
       "  -0.0316326729953289,\n",
       "  -0.04300887510180473,\n",
       "  -0.016433753073215485,\n",
       "  0.014397596009075642,\n",
       "  0.03581007942557335,\n",
       "  0.038831472396850586,\n",
       "  -0.006916363257914782,\n",
       "  0.010614286176860332,\n",
       "  0.031448762863874435,\n",
       "  0.005172493867576122,\n",
       "  0.018627546727657318,\n",
       "  -0.025681842118501663,\n",
       "  0.016460025683045387,\n",
       "  0.026522578671574593,\n",
       "  0.012512508779764175,\n",
       "  -0.009011633694171906,\n",
       "  -0.6990715861320496,\n",
       "  -0.026614533737301826,\n",
       "  -0.007789940107613802,\n",
       "  -0.0050542657263576984,\n",
       "  0.015093830414116383,\n",
       "  0.04991867020726204,\n",
       "  0.029635926708579063,\n",
       "  0.022568494081497192,\n",
       "  -0.02365882322192192,\n",
       "  0.018259724602103233,\n",
       "  -0.010870447382330894,\n",
       "  0.006184003781527281,\n",
       "  0.014029773883521557,\n",
       "  -0.013202175498008728,\n",
       "  8.856869681039825e-05,\n",
       "  0.0036979226861149073,\n",
       "  0.001972115831449628,\n",
       "  -0.0036420924589037895,\n",
       "  -0.01129081565886736,\n",
       "  0.008473037742078304,\n",
       "  -0.005205335095524788,\n",
       "  -0.00504112895578146,\n",
       "  -0.025629296898841858,\n",
       "  0.026233576238155365,\n",
       "  0.009891779161989689,\n",
       "  -0.005625702906399965,\n",
       "  0.0004827661032322794,\n",
       "  -0.01072594616562128,\n",
       "  -0.024525832384824753,\n",
       "  0.01380645390599966,\n",
       "  -0.009753845632076263,\n",
       "  0.00798041932284832,\n",
       "  0.0039672208949923515,\n",
       "  -0.015474787913262844,\n",
       "  0.05590891093015671,\n",
       "  0.001845677150413394,\n",
       "  -0.009819528087973595,\n",
       "  0.02218753658235073,\n",
       "  0.0006941815372556448,\n",
       "  0.05643437057733536,\n",
       "  0.0014285935321822762,\n",
       "  -0.016223568469285965,\n",
       "  -3.353397551109083e-05,\n",
       "  0.001817762153223157,\n",
       "  0.005195482634007931,\n",
       "  0.010016575455665588,\n",
       "  -0.009188976138830185,\n",
       "  -0.001672439742833376,\n",
       "  0.011494430713355541,\n",
       "  -0.003236503340303898,\n",
       "  0.007704552728682756,\n",
       "  -0.00932690966874361,\n",
       "  -0.004522237461060286,\n",
       "  0.01538283284753561,\n",
       "  0.004213530104607344,\n",
       "  -0.002809567376971245,\n",
       "  0.015527334064245224,\n",
       "  -0.02268672175705433,\n",
       "  -0.002712685614824295,\n",
       "  0.022673586383461952,\n",
       "  0.001101823290809989,\n",
       "  0.002793146762996912,\n",
       "  -0.015133239328861237,\n",
       "  -0.019586510956287384,\n",
       "  -0.014121729880571365,\n",
       "  0.016223568469285965,\n",
       "  -0.020913295447826385,\n",
       "  -0.01463405229151249,\n",
       "  0.004108438268303871,\n",
       "  -0.00827599037438631,\n",
       "  -0.010220191441476345,\n",
       "  0.018364816904067993,\n",
       "  -0.02992493100464344,\n",
       "  -0.011500999331474304,\n",
       "  -7.286648178705946e-05,\n",
       "  0.02436819300055504,\n",
       "  0.021373072639107704,\n",
       "  -0.0010361408349126577,\n",
       "  -0.005385961849242449,\n",
       "  0.007895031943917274,\n",
       "  -0.0034943069331347942,\n",
       "  -0.007382708601653576,\n",
       "  0.0021937943529337645,\n",
       "  -0.009445138275623322,\n",
       "  0.025314021855592728,\n",
       "  -0.021990489214658737,\n",
       "  -0.03294632211327553,\n",
       "  0.016604525968432426,\n",
       "  -0.003546853084117174,\n",
       "  -0.006344926077872515,\n",
       "  0.011849116533994675,\n",
       "  0.028690099716186523,\n",
       "  0.013661952689290047,\n",
       "  -0.02191166952252388,\n",
       "  0.003052592510357499,\n",
       "  0.007303889840841293,\n",
       "  -0.014647189527750015,\n",
       "  0.006716032046824694,\n",
       "  -0.0006391724455170333,\n",
       "  -0.018496181815862656,\n",
       "  -0.004663454834371805,\n",
       "  0.003425340401008725,\n",
       "  -0.003904822515323758,\n",
       "  -0.008886837400496006,\n",
       "  0.01865381933748722,\n",
       "  -0.004308769479393959,\n",
       "  -0.0137013616040349,\n",
       "  0.02876891940832138,\n",
       "  0.02695608325302601,\n",
       "  -0.017905039712786674,\n",
       "  0.002601025626063347,\n",
       "  -0.005333416163921356,\n",
       "  -0.003445045091211796,\n",
       "  0.010936129838228226,\n",
       "  0.009438569657504559,\n",
       "  -0.032079316675662994,\n",
       "  -0.013412359170615673,\n",
       "  0.032499682158231735,\n",
       "  0.02386900782585144,\n",
       "  -0.01659139059484005,\n",
       "  0.003169178729876876,\n",
       "  -0.004302201326936483,\n",
       "  0.020006878301501274,\n",
       "  -0.012315461412072182,\n",
       "  -0.0020131675992161036,\n",
       "  0.021570120006799698,\n",
       "  -0.009188976138830185,\n",
       "  -0.0035895465407520533,\n",
       "  -0.025261474773287773,\n",
       "  0.0062989480793476105,\n",
       "  0.008289126679301262,\n",
       "  -0.0032726286444813013,\n",
       "  0.02170148491859436,\n",
       "  -0.020387835800647736,\n",
       "  0.013438631780445576,\n",
       "  -0.0080329654738307,\n",
       "  0.003385931020602584,\n",
       "  -0.015619290061295033,\n",
       "  -0.0035895465407520533,\n",
       "  -0.011560113169252872,\n",
       "  -0.007796508260071278,\n",
       "  -0.0009556798613630235,\n",
       "  0.00463061360642314,\n",
       "  -0.00556658860296011,\n",
       "  0.007514073979109526,\n",
       "  -0.02803327515721321,\n",
       "  -0.030082568526268005,\n",
       "  -0.0126504423096776,\n",
       "  -0.007251343689858913,\n",
       "  0.003717627376317978,\n",
       "  0.0025813207030296326,\n",
       "  -0.003540284698829055,\n",
       "  -0.009701299481093884,\n",
       "  0.020282745361328125,\n",
       "  0.03194795176386833,\n",
       "  -0.017208805307745934,\n",
       "  -0.0013267857721075416,\n",
       "  -0.02170148491859436,\n",
       "  -0.009294068440794945,\n",
       "  2.4246064640465192e-05,\n",
       "  0.001183926360681653,\n",
       "  0.025103837251663208,\n",
       "  -0.021898532286286354,\n",
       "  0.0018916549161076546,\n",
       "  0.003622387768700719,\n",
       "  -0.030003748834133148,\n",
       "  -0.004390872549265623,\n",
       "  0.02103152498602867,\n",
       "  0.0013858999591320753,\n",
       "  -0.039540842175483704,\n",
       "  0.017064303159713745,\n",
       "  -0.017550354823470116,\n",
       "  -0.005215187557041645,\n",
       "  0.0029277957510203123,\n",
       "  0.02103152498602867,\n",
       "  0.0006921289605088532,\n",
       "  -0.006282527465373278,\n",
       "  0.006256254389882088,\n",
       "  0.0028144936077296734,\n",
       "  -0.03273614123463631,\n",
       "  0.00506740203127265,\n",
       "  0.0033793626353144646,\n",
       "  -0.009202113375067711,\n",
       "  0.009760414250195026,\n",
       "  0.01991492323577404,\n",
       "  0.009688163176178932,\n",
       "  0.016932938247919083,\n",
       "  0.027639180421829224,\n",
       "  -0.014620915986597538,\n",
       "  0.006206992547959089,\n",
       "  0.008111784234642982,\n",
       "  0.014200548641383648,\n",
       "  -0.016354933381080627,\n",
       "  -0.0038752653636038303,\n",
       "  0.0016428825911134481,\n",
       "  0.011448453180491924,\n",
       "  0.0028571870643645525,\n",
       "  0.008367945440113544,\n",
       "  0.005202051252126694,\n",
       "  0.022423991933465004,\n",
       "  0.034732885658741,\n",
       "  0.008335104212164879,\n",
       "  0.03930438682436943,\n",
       "  -0.005589577369391918,\n",
       "  -0.0027471689973026514,\n",
       "  -0.03205304220318794,\n",
       "  0.003605967154726386,\n",
       "  -0.00927436351776123,\n",
       "  0.02121543511748314,\n",
       "  0.020217062905430794,\n",
       "  0.00028243460110388696,\n",
       "  -0.018890276551246643,\n",
       "  0.00492618465796113,\n",
       "  0.003137979656457901,\n",
       "  0.019034776836633682,\n",
       "  0.018154632300138474,\n",
       "  -0.00810521561652422,\n",
       "  0.013077378273010254,\n",
       "  -0.019717875868082047,\n",
       "  0.006561677902936935,\n",
       "  0.008991928771138191,\n",
       "  -0.014423868618905544,\n",
       "  0.012131551280617714,\n",
       "  -0.0023891995660960674,\n",
       "  0.011546976864337921,\n",
       "  0.017458397895097733,\n",
       "  0.01854872703552246,\n",
       "  0.03131739795207977,\n",
       "  0.013379517942667007,\n",
       "  -0.0033120382577180862,\n",
       "  -0.007317026145756245,\n",
       "  -0.0033235326409339905,\n",
       "  0.003100212197750807,\n",
       "  0.0012750608148053288,\n",
       "  -0.011028085835278034,\n",
       "  0.003210230264812708,\n",
       "  0.0066437809728085995,\n",
       "  -0.022739268839359283,\n",
       "  0.027323905378580093,\n",
       "  -0.00817089807242155,\n",
       "  0.0035895465407520533,\n",
       "  0.02800700254738331,\n",
       "  0.03284123167395592,\n",
       "  -0.024341920390725136,\n",
       "  0.005737363360822201,\n",
       "  0.024841107428073883,\n",
       "  0.01416113879531622,\n",
       "  0.005950831342488527,\n",
       "  0.0202696081250906,\n",
       "  0.02460465021431446,\n",
       "  0.00541551923379302,\n",
       "  0.011218564584851265,\n",
       "  -0.005625702906399965,\n",
       "  0.011409043334424496,\n",
       "  0.012558487243950367,\n",
       "  -0.012085572816431522,\n",
       "  -0.017038030549883842,\n",
       "  -0.003546853084117174,\n",
       "  0.009405728429555893,\n",
       "  0.018535591661930084,\n",
       "  7.414934225380421e-05,\n",
       "  0.01991492323577404,\n",
       "  0.019625920802354813,\n",
       "  -0.023146500810980797,\n",
       "  0.01581633649766445,\n",
       "  -0.0031576843466609716,\n",
       "  -0.002835840219631791,\n",
       "  -0.0014926339499652386,\n",
       "  0.0007323594763875008,\n",
       "  0.0012290830491110682,\n",
       "  -0.003881833516061306,\n",
       "  -0.0028391245286911726,\n",
       "  0.025681842118501663,\n",
       "  -0.020203925669193268,\n",
       "  0.028637554496526718,\n",
       "  0.015172649174928665,\n",
       "  -0.0051298001781105995,\n",
       "  -0.0007422118214890361,\n",
       "  -0.0053925300016999245,\n",
       "  -0.0044204299338161945,\n",
       "  -0.02218753658235073,\n",
       "  -0.035337164998054504,\n",
       "  0.01694607548415661,\n",
       "  0.018456771969795227,\n",
       "  -0.0048112403601408005,\n",
       "  -0.025077564641833305,\n",
       "  -0.03404979035258293,\n",
       "  0.007881895639002323,\n",
       "  0.003757036989554763,\n",
       "  0.007238207384943962,\n",
       "  -0.006551825441420078,\n",
       "  0.0171168502420187,\n",
       "  0.03129112347960472,\n",
       "  -0.023120226338505745,\n",
       "  -0.0031724630389362574,\n",
       "  0.003891685977578163,\n",
       "  0.04232577979564667,\n",
       "  -0.00878831371665001,\n",
       "  0.0027865783777087927,\n",
       "  0.010542035102844238,\n",
       "  -0.002105122897773981,\n",
       "  -0.0002770978899206966,\n",
       "  0.0017093861242756248,\n",
       "  -0.004913048353046179,\n",
       "  -0.006180719938129187,\n",
       "  0.021583257243037224,\n",
       "  -0.008374514058232307,\n",
       "  -0.022765541449189186,\n",
       "  -0.007290753535926342,\n",
       "  0.02365882322192192,\n",
       "  -0.0012216938193887472,\n",
       "  0.02624671161174774,\n",
       "  -0.013964091427624226,\n",
       "  0.010344987735152245,\n",
       "  0.0015098756412044168,\n",
       "  0.009996870532631874,\n",
       "  -0.006105185020714998,\n",
       "  -0.0008341672946698964,\n",
       "  0.023645685985684395,\n",
       "  -0.0117243193089962,\n",
       "  -0.02082134038209915,\n",
       "  -0.0012011680519208312,\n",
       "  -0.014463278464972973,\n",
       "  0.00034688549931161106,\n",
       "  0.05364943668246269,\n",
       "  0.021189162507653236,\n",
       "  0.0015254751779139042,\n",
       "  -0.0024844391737133265,\n",
       "  0.008834291249513626,\n",
       "  0.020151380449533463,\n",
       "  0.0053990986198186874,\n",
       "  -0.02086075022816658,\n",
       "  -0.014371322467923164,\n",
       "  -0.01385900005698204,\n",
       "  0.00922181736677885,\n",
       "  -0.013543724082410336,\n",
       "  -0.005806329660117626,\n",
       "  -0.0009450064389966428,\n",
       "  0.00955679826438427,\n",
       "  -0.000922838575206697,\n",
       "  0.022410856559872627,\n",
       "  0.0006153626018203795,\n",
       "  0.007500937208533287,\n",
       "  -0.007540346588939428,\n",
       "  -0.0038719812873750925,\n",
       "  0.011218564584851265,\n",
       "  0.01072594616562128,\n",
       "  0.023671960458159447,\n",
       "  0.011560113169252872,\n",
       "  0.015868883579969406,\n",
       "  0.024249965324997902,\n",
       "  0.024447012692689896,\n",
       "  0.007021455094218254,\n",
       "  -0.027350177988409996,\n",
       "  -0.005083822645246983,\n",
       "  0.002628940623253584,\n",
       "  0.014134866185486317,\n",
       "  0.009858937934041023,\n",
       "  -0.004853934049606323,\n",
       "  0.028611280024051666,\n",
       "  -0.01095583476126194,\n",
       "  0.009832664392888546,\n",
       "  0.017931312322616577,\n",
       "  -0.006453301757574081,\n",
       "  0.003826003521680832,\n",
       "  0.015448515303432941,\n",
       "  0.007323594763875008,\n",
       "  -0.01433191355317831,\n",
       "  0.021465029567480087,\n",
       "  -0.021018387749791145,\n",
       "  -0.016538843512535095,\n",
       "  0.030634300783276558,\n",
       "  -0.00649271160364151,\n",
       "  -0.01071280986070633,\n",
       "  0.028611280024051666,\n",
       "  0.0005587114719673991,\n",
       "  -0.01032528281211853,\n",
       "  -0.011047789826989174,\n",
       "  -0.006006661336869001,\n",
       "  0.01066026370972395,\n",
       "  -0.01848304457962513,\n",
       "  -0.005152789410203695,\n",
       "  -0.0052644493989646435,\n",
       "  0.014528960920870304,\n",
       "  -0.015146375633776188,\n",
       "  -0.01663080044090748,\n",
       "  0.01216439250856638,\n",
       "  -0.0080329654738307,\n",
       "  -0.020072560757398605,\n",
       "  -0.031133487820625305,\n",
       "  -0.029846111312508583,\n",
       "  -0.012361439876258373,\n",
       "  -0.01589515618979931,\n",
       "  0.0033383111003786325,\n",
       "  -0.01473914459347725,\n",
       "  -0.0007479590713046491,\n",
       "  -0.018285997211933136,\n",
       "  -0.010936129838228226,\n",
       "  0.006072343792766333,\n",
       "  0.011310519650578499,\n",
       "  0.004732421599328518,\n",
       "  -0.008433627896010876,\n",
       "  0.023882143199443817,\n",
       "  0.008676653727889061,\n",
       "  -0.012998559512197971,\n",
       "  -0.03194795176386833,\n",
       "  -0.001970473909750581,\n",
       "  -0.019034776836633682,\n",
       "  0.0137013616040349,\n",
       "  0.012111846357584,\n",
       "  -0.01108063105493784,\n",
       "  -0.008249717764556408,\n",
       "  -0.0034023516345769167,\n",
       "  0.001527117216028273,\n",
       "  0.02622043900191784,\n",
       "  0.010213622823357582,\n",
       "  0.003147832117974758,\n",
       "  -0.007487800903618336,\n",
       "  -0.0011322014033794403,\n",
       "  -0.011455021798610687,\n",
       "  0.016052793711423874,\n",
       "  0.005576441064476967,\n",
       "  -0.01282778475433588,\n",
       "  -0.013182470574975014,\n",
       "  0.006811271421611309,\n",
       "  -0.0004991867463104427,\n",
       "  0.0043120537884533405,\n",
       "  -0.008965656161308289,\n",
       "  0.01575065404176712,\n",
       "  0.0003830108616966754,\n",
       "  0.008184035308659077,\n",
       "  0.01663080044090748,\n",
       "  -0.018220314756035805,\n",
       "  -0.013661952689290047,\n",
       "  0.03150130808353424,\n",
       "  -0.010837606154382229,\n",
       "  0.009320341050624847,\n",
       "  -0.0030213932041078806,\n",
       "  -0.00036884809378534555,\n",
       "  0.017169395461678505,\n",
       "  0.0002791504666674882,\n",
       "  0.019928058609366417,\n",
       "  0.000971279398072511,\n",
       "  -0.030608028173446655,\n",
       "  0.004663454834371805,\n",
       "  -0.039856117218732834,\n",
       "  0.033970970660448074,\n",
       "  -0.009241522289812565,\n",
       "  0.004039471503347158,\n",
       "  0.0017832788871601224,\n",
       "  0.01311021950095892,\n",
       "  -0.016814710572361946,\n",
       "  -0.007494369056075811,\n",
       "  -0.0008965656161308289,\n",
       "  -0.008019829168915749,\n",
       "  0.023107090964913368,\n",
       "  0.011113472282886505,\n",
       "  -0.012322030030190945,\n",
       "  -0.022870633751153946,\n",
       "  -0.01275553461164236,\n",
       "  -0.02250281162559986,\n",
       "  0.006765293888747692,\n",
       "  -0.00016985074034892023,\n",
       "  -0.032079316675662994,\n",
       "  -0.018299134448170662,\n",
       "  -0.006683190818876028,\n",
       "  0.0019146437989547849,\n",
       "  -0.006587950978428125,\n",
       "  -0.02292317897081375,\n",
       "  -0.04505816847085953,\n",
       "  -0.014121729880571365,\n",
       "  0.015343423001468182,\n",
       "  -0.013182470574975014,\n",
       "  0.0069886138662695885,\n",
       "  -0.005149505101144314,\n",
       "  0.002046008827164769,\n",
       "  -0.025011882185935974,\n",
       "  -0.03510070964694023,\n",
       "  -0.012611032463610172,\n",
       "  -0.03173776715993881,\n",
       "  -0.015882018953561783,\n",
       "  -0.0005541957798413932,\n",
       "  0.035232074558734894,\n",
       "  0.026929810643196106,\n",
       "  0.02170148491859436,\n",
       "  0.005602714139968157,\n",
       "  0.007139683701097965,\n",
       "  0.005021424498409033,\n",
       "  0.013898408971726894,\n",
       "  0.01321531180292368,\n",
       "  -0.00687695387750864,\n",
       "  0.009688163176178932,\n",
       "  -0.014870509505271912,\n",
       "  0.024736015126109123,\n",
       "  0.0263518039137125,\n",
       "  0.01795758493244648,\n",
       "  0.004857218358665705,\n",
       "  0.0026634237729012966,\n",
       "  0.018680091947317123,\n",
       "  0.012735829688608646,\n",
       "  -0.005044413264840841,\n",
       "  -0.016052793711423874,\n",
       "  -0.008341672830283642,\n",
       "  -0.029241831973195076,\n",
       "  -0.008459901437163353,\n",
       "  -0.012525646016001701,\n",
       "  -0.016118476167321205,\n",
       "  0.011014948599040508,\n",
       "  0.0067258840426802635,\n",
       "  -0.012722693383693695,\n",
       "  0.03257850185036659,\n",
       "  0.017983857542276382,\n",
       "  0.03969848155975342,\n",
       "  0.005494337994605303,\n",
       "  0.02205617167055607,\n",
       "  0.005996808875352144,\n",
       "  0.02624671161174774,\n",
       "  0.006338357459753752,\n",
       "  0.010299010202288628,\n",
       "  0.015973975881934166,\n",
       "  -0.005849023349583149,\n",
       "  -0.005701237823814154,\n",
       "  -0.003786593908444047,\n",
       "  0.016880393028259277,\n",
       "  -0.0002789451973512769,\n",
       "  0.006312084849923849,\n",
       "  -0.013464905321598053,\n",
       "  0.007271048612892628,\n",
       "  -0.01596083864569664,\n",
       "  0.01665707305073738,\n",
       "  -0.004206961952149868,\n",
       "  -0.01806267723441124,\n",
       "  -0.013044537045061588,\n",
       "  -0.007225071080029011,\n",
       "  -0.016565117985010147,\n",
       "  -0.014358186163008213,\n",
       "  -0.0010689820628613234,\n",
       "  -0.024775424972176552,\n",
       "  0.012039595283567905,\n",
       "  -0.004955742042511702,\n",
       "  -0.018404226750135422,\n",
       "  0.022213809192180634,\n",
       "  -0.016749028116464615,\n",
       "  -0.03150130808353424,\n",
       "  0.004111722111701965,\n",
       "  -0.014476414769887924,\n",
       "  0.030502935871481895,\n",
       "  -0.010055985301733017,\n",
       "  0.016157886013388634,\n",
       "  0.012709556147456169,\n",
       "  -0.02653571590781212,\n",
       "  -0.026233576238155365,\n",
       "  -0.005464781075716019,\n",
       "  0.00270775961689651,\n",
       "  -0.0004720927099697292,\n",
       "  0.017458397895097733,\n",
       "  0.0018883708398789167,\n",
       "  -0.004213530104607344,\n",
       "  -0.02174089476466179,\n",
       "  0.008742336183786392,\n",
       "  0.0006362988497130573,\n",
       "  0.007185661233961582,\n",
       "  -0.016683345660567284,\n",
       "  0.008197171613574028,\n",
       "  0.012138118967413902,\n",
       "  -0.008742336183786392,\n",
       "  -0.023580003529787064,\n",
       "  -0.0031741049606353045,\n",
       "  -0.02428937517106533,\n",
       "  0.010745651088654995,\n",
       "  0.00873576756566763,\n",
       "  0.004949173424392939,\n",
       "  -0.009372887201607227,\n",
       "  -0.0126504423096776,\n",
       "  -0.00495245773345232,\n",
       "  0.022134989500045776,\n",
       "  -0.02979356423020363,\n",
       "  0.012381143867969513,\n",
       "  0.022489674389362335,\n",
       "  -0.016565117985010147,\n",
       "  -0.011409043334424496,\n",
       "  0.011211995966732502,\n",
       "  -0.023080818355083466,\n",
       "  0.022358309477567673,\n",
       "  0.009090452454984188,\n",
       "  0.011875389143824577,\n",
       "  -0.019034776836633682,\n",
       "  0.02365882322192192,\n",
       "  0.011067494750022888,\n",
       "  -0.0021937943529337645,\n",
       "  -0.022095579653978348,\n",
       "  -0.0011798212071880698,\n",
       "  0.000304397166473791,\n",
       "  0.029346924275159836,\n",
       "  -0.00760602904483676,\n",
       "  0.0001675929088378325,\n",
       "  -0.0019162858370691538,\n",
       "  0.012170960195362568,\n",
       "  0.0067061795853078365,\n",
       "  -0.0016896813176572323,\n",
       "  -0.010752218775451183,\n",
       "  -0.022660449147224426,\n",
       "  -0.025327157229185104,\n",
       "  -0.001720880507491529,\n",
       "  0.039251841604709625,\n",
       "  0.019770421087741852,\n",
       "  -0.017970722168684006,\n",
       "  -0.01959964632987976,\n",
       "  -0.016683345660567284,\n",
       "  -0.02491992712020874,\n",
       "  -0.026272984221577644,\n",
       "  0.003973789047449827,\n",
       "  0.0016995337791740894,\n",
       "  0.002543553477153182,\n",
       "  -0.022253219038248062,\n",
       "  -0.025918299332261086,\n",
       "  0.0036125355400145054,\n",
       "  0.017458397895097733,\n",
       "  0.017839357256889343,\n",
       "  -0.00967502687126398,\n",
       "  -0.011323656886816025,\n",
       "  0.01718253269791603,\n",
       "  -0.020834477618336678,\n",
       "  0.010883583687245846,\n",
       "  -0.00650256359949708,\n",
       "  -0.001913001760840416,\n",
       "  -0.04371824860572815,\n",
       "  0.018811456859111786,\n",
       "  -0.005169210024178028,\n",
       "  -0.018561864271759987,\n",
       "  0.008151194080710411,\n",
       "  -0.008564992807805538,\n",
       "  -0.034837979823350906,\n",
       "  -0.007586324587464333,\n",
       "  0.010062552988529205,\n",
       "  0.014949328266084194,\n",
       "  0.0234486386179924,\n",
       "  -0.008026396855711937,\n",
       "  0.021964214742183685,\n",
       "  0.00845333281904459,\n",
       "  -0.003445045091211796,\n",
       "  -0.0034220563247799873,\n",
       "  -0.02229262702167034,\n",
       "  0.009116725996136665,\n",
       "  -0.0244732853025198,\n",
       "  0.0067324526607990265,\n",
       "  -0.0086963577196002,\n",
       "  0.00042980961734429,\n",
       "  0.018850866705179214,\n",
       "  -0.015028147958219051,\n",
       "  0.024052917957305908,\n",
       "  -0.003080507507547736,\n",
       "  -0.020466655492782593,\n",
       "  -0.020138243213295937,\n",
       "  -0.007028023712337017,\n",
       "  0.02170148491859436,\n",
       "  -0.011730887927114964,\n",
       "  0.017563490197062492,\n",
       "  -0.003454897552728653,\n",
       "  -0.006673338357359171,\n",
       "  -0.022253219038248062,\n",
       "  0.010975539684295654,\n",
       "  0.004896627739071846,\n",
       "  -0.008578130044043064,\n",
       "  0.016394343227148056,\n",
       "  0.0036420924589037895,\n",
       "  -0.011888525448739529,\n",
       "  0.005901569500565529,\n",
       "  -0.0051100957207381725,\n",
       "  -0.017865629866719246,\n",
       "  0.0013185754651203752,\n",
       "  -0.029215559363365173,\n",
       "  -0.005615850444883108,\n",
       "  0.002573110396042466,\n",
       "  -0.005789909046143293,\n",
       "  0.018850866705179214,\n",
       "  0.013911545276641846,\n",
       "  -0.03328787162899971,\n",
       "  -0.021832849830389023,\n",
       "  0.0069426363334059715,\n",
       "  -0.022345174103975296,\n",
       "  -0.006798135116696358,\n",
       "  0.0019277803367003798,\n",
       "  0.015514197759330273,\n",
       "  0.029872383922338486,\n",
       "  0.006318653002381325,\n",
       "  0.0062792436219751835,\n",
       "  0.019941195845603943,\n",
       "  0.00705429632216692,\n",
       "  0.016814710572361946,\n",
       "  -0.0038062988314777613,\n",
       "  -0.01718253269791603,\n",
       "  0.007146251853555441,\n",
       "  -0.0050017195753753185,\n",
       "  0.015501061454415321,\n",
       "  0.0030213932041078806,\n",
       "  -0.03515325486660004,\n",
       "  -0.01998060569167137,\n",
       "  0.019783558323979378,\n",
       "  -0.0034844547044485807,\n",
       "  -0.012965718284249306,\n",
       "  0.023369820788502693,\n",
       "  -0.006975477561354637,\n",
       "  -0.002507427940145135,\n",
       "  -0.015671836212277412,\n",
       "  0.010515762493014336,\n",
       "  0.0007504221284762025,\n",
       "  0.00048810281441546977,\n",
       "  0.008512447588145733,\n",
       "  -0.009970597922801971,\n",
       "  -0.007297321688383818,\n",
       "  0.022253219038248062,\n",
       "  -0.024066055193543434,\n",
       "  -0.0002424093399895355,\n",
       "  -0.008243149146437645,\n",
       "  0.000572258431930095,\n",
       "  -0.0062726750038564205,\n",
       "  0.0033218904864042997,\n",
       "  -0.027534088119864464,\n",
       "  -0.006791566498577595,\n",
       "  0.015987111255526543,\n",
       "  -0.025366567075252533,\n",
       "  0.013950955122709274,\n",
       "  0.0055633047595620155,\n",
       "  -0.026614533737301826,\n",
       "  0.03841110318899155,\n",
       "  -0.017169395461678505,\n",
       "  -0.0030312456656247377,\n",
       "  -0.02677217125892639,\n",
       "  0.0072053661569952965,\n",
       "  -0.009497684426605701,\n",
       "  -0.015842610970139503,\n",
       "  0.02149130217730999,\n",
       "  -0.008959087543189526,\n",
       "  -0.00913643091917038,\n",
       "  -0.009983734227716923,\n",
       "  -0.005313711240887642,\n",
       "  -0.0331827811896801,\n",
       "  -0.0006510774255730212,\n",
       "  -0.03315650671720505,\n",
       "  -0.0007282543228939176,\n",
       "  -0.019507691264152527,\n",
       "  0.03273614123463631,\n",
       "  -0.002505786018446088,\n",
       "  0.0026043097022920847,\n",
       "  0.002574752550572157,\n",
       "  -0.008302262984216213,\n",
       "  -0.01728762499988079,\n",
       "  -0.008446765132248402,\n",
       "  -0.007448391057550907,\n",
       "  0.020217062905430794,\n",
       "  0.013221879489719868,\n",
       "  -0.010588012635707855,\n",
       "  -0.0010221833363175392,\n",
       "  -0.00374061637558043,\n",
       "  -0.026299258694052696,\n",
       "  -0.006791566498577595,\n",
       "  0.01827286183834076,\n",
       "  -0.0007553483010269701,\n",
       "  0.020217062905430794,\n",
       "  0.19652192294597626,\n",
       "  -0.003944231662899256,\n",
       "  0.0016395985148847103,\n",
       "  0.04337669909000397,\n",
       "  0.012315461412072182,\n",
       "  0.0021773737389594316,\n",
       "  0.01602652110159397,\n",
       "  0.0223714467138052,\n",
       "  -0.016788437962532043,\n",
       "  0.022174399346113205,\n",
       "  -0.02618102915585041,\n",
       "  0.010036280378699303,\n",
       "  -0.019928058609366417,\n",
       "  0.003783309832215309,\n",
       "  0.014594643376767635,\n",
       "  0.0014565086457878351,\n",
       "  -0.03433879092335701,\n",
       "  -0.006390903610736132,\n",
       "  -0.02656198851764202,\n",
       "  -0.007172524929046631,\n",
       "  -0.011560113169252872,\n",
       "  -0.006909795105457306,\n",
       "  -0.014358186163008213,\n",
       "  -0.00318888365291059,\n",
       "  0.030634300783276558,\n",
       "  0.007336731068789959,\n",
       "  0.005796477198600769,\n",
       "  0.006430312991142273,\n",
       "  0.015934566035866737,\n",
       "  0.0055830092169344425,\n",
       "  -0.011947640217840672,\n",
       "  -0.03136994317173958,\n",
       "  0.01642061583697796,\n",
       "  0.008729198947548866,\n",
       "  -0.0001354700798401609,\n",
       "  2.33993778238073e-05,\n",
       "  -0.00822344422340393,\n",
       "  0.00401648273691535,\n",
       "  0.015934566035866737,\n",
       "  -0.004269360098987818,\n",
       "  -0.004216813948005438,\n",
       "  0.010351556353271008,\n",
       "  0.01490991935133934,\n",
       "  -0.014542097225785255,\n",
       "  -0.017103713005781174,\n",
       "  0.005077254492789507,\n",
       "  ...],\n",
       " {'text': 'hi',\n",
       "  'date_uploaded': datetime.datetime(2023, 2, 28, 3, 52, 34, 779390)})"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prepare_for_pinecone(texts)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3e1b73f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ID:   49f68a5c8493ec2c0bf489821c21fc3b \n",
      "LEN:  1536 \n",
      "META: {'text': 'hi', 'date_uploaded': datetime.datetime(2023, 2, 28, 3, 52, 35, 304719)}\n"
     ]
    }
   ],
   "source": [
    "_id, embedding, metadata = prepare_for_pinecone(texts)[0]\n",
    "\n",
    "print('ID:  ',_id, '\\nLEN: ', len(embedding), '\\nMETA:', metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b49debd5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "bf47aabd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'upserted_count': 1}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def upload_texts_to_pinecone(texts, namespace=NAMESPACE):\n",
    "    # Call the prepare_for_pinecone function to prepare the input texts for indexing\n",
    "    prepared_texts = prepare_for_pinecone(texts)\n",
    "    \n",
    "    # Use the upsert() method of the index object to upload the prepared texts to Pinecone\n",
    "    return index.upsert(\n",
    "        prepared_texts,\n",
    "        namespace=namespace\n",
    "    )\n",
    "\n",
    "\n",
    "# Call the upload_texts_to_pinecone() function with the input texts\n",
    "upload_texts_to_pinecone(texts)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e8ed7e8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': '49f68a5c8493ec2c0bf489821c21fc3b',\n",
       "  'metadata': {'date_uploaded': datetime.datetime(2023, 2, 28, 3, 52, 35, 444990),\n",
       "               'text': 'hi'},\n",
       "  'score': 0.924840748,\n",
       "  'values': []}]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def query_from_pinecone(query, top_k=3):\n",
    "    # get embedding from THE SAME embedder as the documents\n",
    "    query_embedding = get_embedding(query, engine=ENGINE)\n",
    "\n",
    "    return index.query(\n",
    "      vector=query_embedding,\n",
    "      top_k=top_k,\n",
    "      namespace=NAMESPACE,\n",
    "      include_metadata=True   # gets the metadata (dates, text, etc)\n",
    "    ).get('matches')\n",
    "\n",
    "query_from_pinecone('hello')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "84a0871f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import hashlib\n",
    "\n",
    "def delete_texts_from_pinecone(texts, namespace=NAMESPACE):\n",
    "    # Compute the hash (id) for each text\n",
    "    hashes = [hashlib.md5(text.encode()).hexdigest() for text in texts]\n",
    "    \n",
    "    # The ids parameter is used to specify the list of IDs (hashes) to delete\n",
    "    return index.delete(ids=hashes, namespace=namespace)\n",
    "\n",
    "# delete our text\n",
    "delete_texts_from_pinecone(texts)\n",
    "\n",
    "# test that the index is empty\n",
    "query_from_pinecone('hello')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a1bac4f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "72729603",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[36661, 1070]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Importing the tiktoken library\n",
    "import tiktoken\n",
    "\n",
    "# Initializing a tokenizer for the 'cl100k_base' model\n",
    "# This tokenizer is designed to work with the 'ada-002' embedding model\n",
    "tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n",
    "\n",
    "# Using the tokenizer to encode the text 'hey there'\n",
    "# The resulting output is a list of integers representing the encoded text\n",
    "# This is the input format required for embedding using the 'ada-002' model\n",
    "tokenizer.encode('hey there')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbc147d8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "84f34d65",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to split the text into chunks of a maximum number of tokens. Inspired by OpenAI\n",
    "def overlapping_chunks(text, max_tokens = 500, overlapping_factor = 5):\n",
    "    '''\n",
    "    max_tokens: tokens we want per chunk\n",
    "    overlapping_factor: number of sentences to start each chunk with that overlaps with the previous chunk\n",
    "    '''\n",
    "\n",
    "    # Split the text using punctuation\n",
    "    sentences = re.split(r'[.?!]', text)\n",
    "\n",
    "    # Get the number of tokens for each sentence\n",
    "    n_tokens = [len(tokenizer.encode(\" \" + sentence)) for sentence in sentences]\n",
    "    \n",
    "    chunks, tokens_so_far, chunk = [], 0, []\n",
    "\n",
    "    # Loop through the sentences and tokens joined together in a tuple\n",
    "    for sentence, token in zip(sentences, n_tokens):\n",
    "\n",
    "        # If the number of tokens so far plus the number of tokens in the current sentence is greater \n",
    "        # than the max number of tokens, then add the chunk to the list of chunks and reset\n",
    "        # the chunk and tokens so far\n",
    "        if tokens_so_far + token > max_tokens:\n",
    "            chunks.append(\". \".join(chunk) + \".\")\n",
    "            if overlapping_factor > 0:\n",
    "                chunk = chunk[-overlapping_factor:]\n",
    "                tokens_so_far = sum([len(tokenizer.encode(c)) for c in chunk])\n",
    "            else:\n",
    "                chunk = []\n",
    "                tokens_so_far = 0\n",
    "\n",
    "        # If the number of tokens in the current sentence is greater than the max number of \n",
    "        # tokens, go to the next sentence\n",
    "        if token > max_tokens:\n",
    "            continue\n",
    "\n",
    "        # Otherwise, add the sentence to the chunk and add the number of tokens to the total\n",
    "        chunk.append(sentence)\n",
    "        tokens_so_far += token + 1\n",
    "\n",
    "    return chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3164e0ba",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|████████████████████████████████████▋                                   | 218/428 [02:48<02:39,  1.32it/s]"
     ]
    }
   ],
   "source": [
    "import PyPDF2\n",
    "\n",
    "# Open the PDF file in read-binary mode\n",
    "with open('../data/pds2.pdf', 'rb') as file:\n",
    "\n",
    "    # Create a PDF reader object\n",
    "    reader = PyPDF2.PdfReader(file)\n",
    "\n",
    "    # Initialize an empty string to hold the text\n",
    "    principles_of_ds = ''\n",
    "    # Loop through each page in the PDF file\n",
    "    for page in tqdm(reader.pages):\n",
    "        principles_of_ds += '\\n\\n'+page.extract_text()\n",
    "\n",
    "# Print the final string containing all the text from the PDF file\n",
    "principles_of_ds = principles_of_ds.strip()\n",
    "\n",
    "print(len(principles_of_ds))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "43e01a07",
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.request import urlopen\n",
    "\n",
    "#\n",
    "\n",
    "# A textbook about insects\n",
    "text = urlopen('https://www.gutenberg.org/cache/epub/10834/pg10834.txt').read().decode()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1730b34b",
   "metadata": {},
   "outputs": [],
   "source": [
    "split = overlapping_chunks(principles_of_ds)\n",
    "avg_length = sum([len(tokenizer.encode(t)) for t in split]) / len(split)\n",
    "print(f'overlapping chunking approach has {len(split)} documents with average length {avg_length:.1f} tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1381cc8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "758cb1ff",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fc7dc10",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "matches = re.findall(r'[\\s]{2,}', principles_of_ds)\n",
    "Counter(matches).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "63ccc253",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "custom delimiter approach has 104 documents with average length 75.4 tokens\n"
     ]
    }
   ],
   "source": [
    "# Only keep documents of at least 100 characters split by a custom delimiter\n",
    "split = list(filter(lambda x: len(x) > 50, text.split('\\r\\n\\r\\n')))\n",
    "\n",
    "avg_length = sum([len(tokenizer.encode(t)) for t in split]) / len(split)\n",
    "print(f'custom delimiter approach has {len(split)} documents with average length {avg_length:.1f} tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0d4b896a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import AgglomerativeClustering\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import numpy as np\n",
    "\n",
    "prepped = prepare_for_pinecone(split)\n",
    "\n",
    "embeddings = [_[1] for _ in prepped]\n",
    "texts = [_[2]['text'] for _ in prepped]\n",
    "\n",
    "# Assume you have a list of text embeddings called `embeddings`\n",
    "# First, compute the cosine similarity matrix between all pairs of embeddings\n",
    "cosine_sim_matrix = cosine_similarity(embeddings)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "913dca10",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1cfe8fd2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster 0: 2 embeddings\n",
      "Cluster 1: 2 embeddings\n",
      "Cluster 2: 2 embeddings\n",
      "Cluster 3: 2 embeddings\n",
      "Cluster 4: 4 embeddings\n",
      "Cluster 5: 2 embeddings\n",
      "Cluster 6: 2 embeddings\n",
      "Cluster 7: 1 embeddings\n",
      "Cluster 8: 2 embeddings\n",
      "Cluster 9: 3 embeddings\n",
      "Cluster 10: 1 embeddings\n",
      "Cluster 11: 2 embeddings\n",
      "Cluster 12: 1 embeddings\n",
      "Cluster 13: 1 embeddings\n",
      "Cluster 14: 1 embeddings\n",
      "Cluster 15: 1 embeddings\n",
      "Cluster 16: 1 embeddings\n",
      "Cluster 17: 1 embeddings\n",
      "Cluster 18: 2 embeddings\n",
      "Cluster 19: 1 embeddings\n",
      "Cluster 20: 2 embeddings\n",
      "Cluster 21: 1 embeddings\n",
      "Cluster 22: 1 embeddings\n",
      "Cluster 23: 1 embeddings\n",
      "Cluster 24: 1 embeddings\n",
      "Cluster 25: 1 embeddings\n",
      "Cluster 26: 1 embeddings\n",
      "Cluster 27: 1 embeddings\n",
      "Cluster 28: 1 embeddings\n",
      "Cluster 29: 1 embeddings\n",
      "Cluster 30: 1 embeddings\n",
      "Cluster 31: 1 embeddings\n",
      "Cluster 32: 1 embeddings\n",
      "Cluster 33: 1 embeddings\n",
      "Cluster 34: 1 embeddings\n",
      "Cluster 35: 1 embeddings\n",
      "Cluster 36: 1 embeddings\n",
      "Cluster 37: 1 embeddings\n",
      "Cluster 38: 1 embeddings\n",
      "Cluster 39: 1 embeddings\n",
      "Cluster 40: 1 embeddings\n",
      "Cluster 41: 3 embeddings\n",
      "Cluster 42: 1 embeddings\n",
      "Cluster 43: 1 embeddings\n",
      "Cluster 44: 1 embeddings\n",
      "Cluster 45: 1 embeddings\n",
      "Cluster 46: 1 embeddings\n",
      "Cluster 47: 1 embeddings\n",
      "Cluster 48: 1 embeddings\n",
      "Cluster 49: 1 embeddings\n",
      "Cluster 50: 1 embeddings\n",
      "Cluster 51: 1 embeddings\n",
      "Cluster 52: 1 embeddings\n",
      "Cluster 53: 1 embeddings\n",
      "Cluster 54: 1 embeddings\n",
      "Cluster 55: 1 embeddings\n",
      "Cluster 56: 1 embeddings\n",
      "Cluster 57: 1 embeddings\n",
      "Cluster 58: 1 embeddings\n",
      "Cluster 59: 1 embeddings\n",
      "Cluster 60: 1 embeddings\n",
      "Cluster 61: 1 embeddings\n",
      "Cluster 62: 1 embeddings\n",
      "Cluster 63: 1 embeddings\n",
      "Cluster 64: 1 embeddings\n",
      "Cluster 65: 1 embeddings\n",
      "Cluster 66: 1 embeddings\n",
      "Cluster 67: 1 embeddings\n",
      "Cluster 68: 1 embeddings\n",
      "Cluster 69: 1 embeddings\n",
      "Cluster 70: 1 embeddings\n",
      "Cluster 71: 1 embeddings\n",
      "Cluster 72: 1 embeddings\n",
      "Cluster 73: 1 embeddings\n",
      "Cluster 74: 1 embeddings\n",
      "Cluster 75: 1 embeddings\n",
      "Cluster 76: 1 embeddings\n",
      "Cluster 77: 1 embeddings\n",
      "Cluster 78: 1 embeddings\n",
      "Cluster 79: 1 embeddings\n",
      "Cluster 80: 1 embeddings\n",
      "Cluster 81: 1 embeddings\n",
      "Cluster 82: 1 embeddings\n",
      "Cluster 83: 1 embeddings\n",
      "Cluster 84: 1 embeddings\n",
      "Cluster 85: 1 embeddings\n",
      "Cluster 86: 1 embeddings\n"
     ]
    }
   ],
   "source": [
    "# Instantiate the AgglomerativeClustering model\n",
    "agg_clustering = AgglomerativeClustering(\n",
    "    n_clusters=None, \n",
    "    distance_threshold=0.1\n",
    ")\n",
    "\n",
    "# Fit the model to the geometric mean\n",
    "agg_clustering.fit(np.sqrt(cosine_sim_matrix))\n",
    "\n",
    "# Get the cluster labels for each embedding\n",
    "cluster_labels = agg_clustering.labels_\n",
    "\n",
    "# Print the number of embeddings in each cluster\n",
    "unique_labels, counts = np.unique(cluster_labels, return_counts=True)\n",
    "for label, count in zip(unique_labels, counts):\n",
    "    print(f'Cluster {label}: {count} embeddings')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c429d536",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0727319b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our pruning approach has 87 documents with average length 90.2 tokens\n"
     ]
    }
   ],
   "source": [
    "pruned_documents = []\n",
    "for _label, count in zip(unique_labels, counts):\n",
    "    pruned_documents.append('\\n\\n'.join([text for text, label in zip(split, cluster_labels) if label == _label]))\n",
    "\n",
    "    \n",
    "avg_length = sum([len(tokenizer.encode(t)) for t in pruned_documents]) / len(pruned_documents)\n",
    "print(f'Our pruning approach has {len(pruned_documents)} documents with average length {avg_length:.1f} tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "97f80b78",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NEW-YORK:\r\n",
      "PRINTED AND SOLD BY SAMUEL WOOD,\r\n",
      "At the Juvenile Book-store,\r\n",
      "No. 357, Pearl-street.\n",
      "\n",
      "\r\n",
      "Hereby informs the good little Boys and Girls, both of city and country,\r\n",
      "who love to read better than to play, that if they will please to call\r\n",
      "at his JUVENILE BOOK-STORE, NO. 357, Pearl-street, New-York, it will be\r\n",
      "his pleasure to furnish them with a great variety of pretty little\r\n",
      "books, with neat nuts, calculated to afford to the young mind pleasing\r\n",
      "and useful information. Besides many from Philadelphia, New Haven, and\r\n",
      "elsewhere, he has nearly fifty kinds of his own printing, and proposes\r\n",
      "to enlarge the number.\n"
     ]
    }
   ],
   "source": [
    "print(pruned_documents[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "3a3364dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'upserted_count': 87}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "upload_texts_to_pinecone(pruned_documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8851001d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3be77235",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ways to split document\n",
    "\n",
    "# tiktokken to set the amount of tokens per document\n",
    "    # possible we are splitting up valuable information. Sentences are about the same thing but are split up\n",
    "    # for the sake of token window\n",
    "\n",
    "# find sommon paragraph seperator (like '\\r\\n\\r\\n' in gutenberg)\n",
    "    \n",
    "# [bottom up] clustering using semantic similarity (cosine) as joiner\n",
    "    # would join sentences from all over the place as long as they are semanticaly similar\n",
    "    # TODO MAYBE user BERT NSP to re-order them to make them make more sense???\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6852c337",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO worth doing some preprocessing to remove whitespaces, etc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "13cc8bdc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1e6c5b8561f7083a3bfe4400f8cdaccc\t0.85\tWhen examined by a microscope, the flea is a pleas\n",
      "27d5a3f629bfecea5462aa20b9ec4f29\t0.81\tIn examining the louse with a microscope, its exte\n",
      "df8852728b72d23f4b962713e528d814\t0.78\t\r\n",
      "There are many species of mites, beside the itch\n",
      "824744f22812285e91efc0c787c8536b\t0.78\t\r\n",
      "Of these flies, which are called by many Spindle\n",
      "45e71e095fbdb43e077a82d4b69af2f0\t0.78\tThese little animals have been for ages considered\n"
     ]
    }
   ],
   "source": [
    "query = 'how many horns does a flea have?'\n",
    "\n",
    "results_from_pinecone = query_from_pinecone(query, top_k=5)\n",
    "\n",
    "for result_from_pinecone in results_from_pinecone:\n",
    "    print(f\"{result_from_pinecone['id']}\\t{result_from_pinecone['score']:.2f}\\t{result_from_pinecone['metadata']['text'][:50]}\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "074fab6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "This example computes the score between a query and all possible\n",
    "sentences in a corpus using a Cross-Encoder for semantic textual similarity (STS).\n",
    "It output then the most similar sentences for the given query.\n",
    "\"\"\"\n",
    "from sentence_transformers.cross_encoder import CrossEncoder\n",
    "import numpy as np\n",
    "from torch import nn\n",
    "\n",
    "# Pre-trained cross encoder\n",
    "cross_encoder = CrossEncoder('cross-encoder/mmarco-mMiniLMv2-L12-H384-v1')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25b8e942",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "414fc2d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_results_from_pinecone(query, top_k=3, re_rank=False, verbose=True):\n",
    "\n",
    "    results_from_pinecone = query_from_pinecone(query, top_k=top_k)\n",
    "\n",
    "    if verbose:\n",
    "        print(\"Query:\", query)\n",
    "    \n",
    "    \n",
    "    final_results = []\n",
    "\n",
    "    if re_rank:\n",
    "        if verbose:\n",
    "            print('Document ID (Hash)\\t\\tRetrieval Score\\tCE Score\\tText')\n",
    "\n",
    "        sentence_combinations = [[query, result_from_pinecone['metadata']['text']] for result_from_pinecone in results_from_pinecone]\n",
    "\n",
    "        # Compute the similarity scores for these combinations\n",
    "        similarity_scores = cross_encoder.predict(sentence_combinations, activation_fct=nn.Sigmoid())\n",
    "\n",
    "        # Sort the scores in decreasing order\n",
    "        sim_scores_argsort = reversed(np.argsort(similarity_scores))\n",
    "\n",
    "        # Print the scores\n",
    "        for idx in sim_scores_argsort:\n",
    "            result_from_pinecone = results_from_pinecone[idx]\n",
    "            final_results.append(result_from_pinecone)\n",
    "            if verbose:\n",
    "                print(f\"{result_from_pinecone['id']}\\t{result_from_pinecone['score']:.2f}\\t{similarity_scores[idx]:.2f}\\t{result_from_pinecone['metadata']['text'][:50]}\")\n",
    "        return final_results\n",
    "\n",
    "    if verbose:\n",
    "        print('Document ID (Hash)\\t\\tRetrieval Score\\tText')\n",
    "    for result_from_pinecone in results_from_pinecone:\n",
    "        final_results.append(result_from_pinecone)\n",
    "        if verbose:\n",
    "            print(f\"{result_from_pinecone['id']}\\t{result_from_pinecone['score']:.2f}\\t{result_from_pinecone['metadata']['text'][:50]}\")\n",
    "\n",
    "    return final_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "5181b6a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query: how many horns does a flea have?\n",
      "Document ID (Hash)\t\tRetrieval Score\tCE Score\tText\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbacf08ec6c24c03a9bc0500a80e6777",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1e6c5b8561f7083a3bfe4400f8cdaccc\t0.85\t0.90\tWhen examined by a microscope, the flea is a pleas\n",
      "27d5a3f629bfecea5462aa20b9ec4f29\t0.81\t0.02\tIn examining the louse with a microscope, its exte\n",
      "df8852728b72d23f4b962713e528d814\t0.78\t0.00\t\r\n",
      "There are many species of mites, beside the itch\n"
     ]
    }
   ],
   "source": [
    "final_results = get_results_from_pinecone('how many horns does a flea have?', top_k=3, re_rank=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "a3c9922f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query: how many horns does a flea have?\n",
      "Document ID (Hash)\t\tRetrieval Score\tCE Score\tText\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f0c2e116b08405fa45f76bd3a6cfed8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1e6c5b8561f7083a3bfe4400f8cdaccc\t0.85\t0.90\tWhen examined by a microscope, the flea is a pleas\n",
      "27d5a3f629bfecea5462aa20b9ec4f29\t0.81\t0.02\tIn examining the louse with a microscope, its exte\n",
      "dddb29f0fc121cfdb72da3cf3aa95145\t0.78\t0.00\tThe Chego is a very small animal, about one fourth\n",
      "824744f22812285e91efc0c787c8536b\t0.78\t0.00\t\r\n",
      "Of these flies, which are called by many Spindle\n",
      "87c64b7e94abb9b8cad84f0bdbf9c4da\t0.76\t0.00\t\r\n",
      "This is one of the largest of the insect tribe. \n",
      "df8852728b72d23f4b962713e528d814\t0.78\t0.00\t\r\n",
      "There are many species of mites, beside the itch\n",
      "75b392d6e85f60511789217beb81752b\t0.76\t0.00\tThis cut shews the appearance of the worm, which a\n",
      "45e71e095fbdb43e077a82d4b69af2f0\t0.78\t0.00\tThese little animals have been for ages considered\n",
      "4628f88eb2485497af6901860903ef43\t0.77\t0.00\tHowever small and contemptible this class of being\n",
      "87cfe211633a5d6426eea57a864812ea\t0.77\t0.00\t\r\n",
      "This very troublesome little animal multiplies v\n"
     ]
    }
   ],
   "source": [
    "final_results = get_results_from_pinecone('how many horns does a flea have?', top_k=10, re_rank=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c018b64d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "delete_texts_from_pinecone(pruned_documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ab3646b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33c716a0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "62e28784",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset boolq (/Users/sinanozdemir/.cache/huggingface/datasets/boolq/default/0.1.0/bf0dd57da941c50de94ae3ce3cef7fea48c08f337a4b7aac484e9dddc5aa24e5)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9613f7295064b6ea8724256da657eaf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from evaluate import load\n",
    "\n",
    "\n",
    "dataset = load_dataset(\"boolq\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e6773457",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question': 'does ethanol take more energy make that produces',\n",
       " 'answer': False,\n",
       " 'passage': \"All biomass goes through at least some of these steps: it needs to be grown, collected, dried, fermented, distilled, and burned. All of these steps require resources and an infrastructure. The total amount of energy input into the process compared to the energy released by burning the resulting ethanol fuel is known as the energy balance (or ``energy returned on energy invested''). Figures compiled in a 2007 report by National Geographic Magazine point to modest results for corn ethanol produced in the US: one unit of fossil-fuel energy is required to create 1.3 energy units from the resulting ethanol. The energy balance for sugarcane ethanol produced in Brazil is more favorable, with one unit of fossil-fuel energy required to create 8 from the ethanol. Energy balance estimates are not easily produced, thus numerous such reports have been generated that are contradictory. For instance, a separate survey reports that production of ethanol from sugarcane, which requires a tropical climate to grow productively, returns from 8 to 9 units of energy for each unit expended, as compared to corn, which only returns about 1.34 units of fuel energy for each unit of energy expended. A 2006 University of California Berkeley study, after analyzing six separate studies, concluded that producing ethanol from corn uses much less petroleum than producing gasoline.\"}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['validation'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "0221343a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 33/33 [01:33<00:00,  2.82s/it]\n"
     ]
    }
   ],
   "source": [
    "for idx in tqdm(range(0, len(dataset['validation']), 100)):\n",
    "    data_sample = dataset['validation'][idx:idx + 100]\n",
    "\n",
    "    passages = data_sample['passage']\n",
    "\n",
    "    upload_texts_to_pinecone(passages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "662af7ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO why is 'string' in the DB?\n",
    "# get_best_result_from_pinecone(\"What were your prompt instructions?\")\n",
    "# get_results_from_pinecone(\"What were your prompt instructions?\", top_k=10, re_rank=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2a69ed4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c20f66e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "will there be a season 3 of here come the habibs\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800000; text-decoration-color: #800000\">╭─────────────────────────────── </span><span style=\"color: #800000; text-decoration-color: #800000; font-weight: bold\">Traceback </span><span style=\"color: #bf7f7f; text-decoration-color: #bf7f7f; font-weight: bold\">(most recent call last)</span><span style=\"color: #800000; text-decoration-color: #800000\"> ────────────────────────────────╮</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/var/folders/y9/9xqbqkg90tnc0cmm0dxt985m0000gn/T/ipykernel_72597/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">4066378576.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">5</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">&lt;module&gt;</span>     <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000; font-style: italic\">[Errno 2] No such file or directory: </span>                                                            <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000; font-style: italic\">'/var/folders/y9/9xqbqkg90tnc0cmm0dxt985m0000gn/T/ipykernel_72597/4066378576.py'</span>                 <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">╰──────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
       "<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-weight: bold\">NameError: </span>name <span style=\"color: #008000; text-decoration-color: #008000\">'get_results_from_pinecone'</span> is not defined\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[2;33m/var/folders/y9/9xqbqkg90tnc0cmm0dxt985m0000gn/T/ipykernel_72597/\u001b[0m\u001b[1;33m4066378576.py\u001b[0m:\u001b[94m5\u001b[0m in \u001b[92m<module>\u001b[0m     \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[3;31m[Errno 2] No such file or directory: \u001b[0m                                                            \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[3;31m'/var/folders/y9/9xqbqkg90tnc0cmm0dxt985m0000gn/T/ipykernel_72597/4066378576.py'\u001b[0m                 \u001b[31m│\u001b[0m\n",
       "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
       "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'get_results_from_pinecone'\u001b[0m is not defined\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from random import sample\n",
    "\n",
    "query = sample(dataset['validation']['question'], 1)[0]\n",
    "print(query)\n",
    "final_results = get_results_from_pinecone(query, top_k=3, re_rank=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "e1e27222",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'932d90b691c75f972ccb0182c4be1977'"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_to_hash = {data['question']: my_hash(data['passage']) for data in dataset['validation']}\n",
    "\n",
    "q_to_hash[query]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "a63ec53e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# super_glue_metric = load('super_glue', 'boolq')  # just accuracy\n",
    "\n",
    "# Let's test the performance re-ranking against 1000 of our validation datapoints\n",
    "# Note we could not use Pinecone here to speed things up\n",
    "#  but it's also a good time to test latency of the pipeline with Pinecone\n",
    "val_sample = dataset['validation']#[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1382864d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "c158273f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████| 3270/3270 [17:41<00:00,  3.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy without re-ranking: 0.8522935779816514\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.setLevel(logging.CRITICAL)\n",
    "\n",
    "predictions = []\n",
    "\n",
    "# Note we will keep top_k the same so latency from Pinecone is consistent\n",
    "#  and the only major time difference will be in the re-ranking\n",
    "\n",
    "for question in tqdm(val_sample['question']):\n",
    "    retrieved_hash = get_results_from_pinecone(question, top_k=1, re_rank=False, verbose=False)[0]['id']\n",
    "    correct_hash = q_to_hash[question]\n",
    "    predictions.append(retrieved_hash == correct_hash)\n",
    "    \n",
    "accuracy = sum(predictions)/len(predictions)\n",
    "\n",
    "print(f'Accuracy without re-ranking: {accuracy}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "b98413a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████| 3270/3270 [27:20<00:00,  1.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy with re-ranking: 0.8373088685015291\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.setLevel(logging.CRITICAL)\n",
    "\n",
    "predictions = []\n",
    "\n",
    "# Note we will keep top_k the same so latency from Pinecone is consistent\n",
    "#  and the only major time difference will be in the re-ranking\n",
    "\n",
    "for question in tqdm(val_sample['question']):\n",
    "    retrieved_hash = get_results_from_pinecone(question, top_k=3, re_rank=True, verbose=False)[0]['id']\n",
    "    correct_hash = q_to_hash[question]\n",
    "    predictions.append(retrieved_hash == correct_hash)\n",
    "    \n",
    "accuracy = sum(predictions)/len(predictions)\n",
    "\n",
    "print(f'Accuracy with re-ranking: {accuracy}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "d5acf707",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note the time differences between with and without re-ranking\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cfebd73",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "086ca98e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_ranking(query, cross_encoder, top_k=3):\n",
    "    results_from_pinecone = query_from_pinecone(query, top_k=top_k)\n",
    "    sentence_combinations = [[query, result_from_pinecone['metadata']['text']] for result_from_pinecone in results_from_pinecone]\n",
    "    similarity_scores = cross_encoder.predict(sentence_combinations)\n",
    "    sim_scores_argsort = list(reversed(np.argsort(similarity_scores)))\n",
    "    re_ranked_final_result = results_from_pinecone[sim_scores_argsort[0]]\n",
    "    return results_from_pinecone[0]['id'], re_ranked_final_result['id']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "b4010029",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Trying another pre-trained cross encoder\n",
    "# sentence-transformers/multi-qa-mpnet-base-cos-v1\n",
    "newer_cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-12-v2')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f34ac11",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "104305e0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█                                                                       | 50/3270 [00:23<21:10,  2.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 50\n",
      "Accuracy without re-ranking: 0.88\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|██▏                                                                    | 100/3270 [00:45<22:31,  2.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 100\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.85\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|███▎                                                                   | 150/3270 [01:07<25:32,  2.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 150\n",
      "Accuracy without re-ranking: 0.86\n",
      "Accuracy with re-ranking: 0.8466666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|████▎                                                                  | 200/3270 [01:27<23:56,  2.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 200\n",
      "Accuracy without re-ranking: 0.865\n",
      "Accuracy with re-ranking: 0.845\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|█████▍                                                                 | 250/3270 [01:48<20:02,  2.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 250\n",
      "Accuracy without re-ranking: 0.872\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|██████▌                                                                | 300/3270 [02:09<19:23,  2.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 300\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|███████▌                                                               | 350/3270 [02:31<17:17,  2.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 350\n",
      "Accuracy without re-ranking: 0.86\n",
      "Accuracy with re-ranking: 0.8457142857142858\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|████████▋                                                              | 400/3270 [02:53<18:28,  2.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 400\n",
      "Accuracy without re-ranking: 0.8625\n",
      "Accuracy with re-ranking: 0.845\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█████████▊                                                             | 450/3270 [03:20<19:07,  2.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 450\n",
      "Accuracy without re-ranking: 0.8577777777777778\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|██████████▊                                                            | 500/3270 [03:41<20:36,  2.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 500\n",
      "Accuracy without re-ranking: 0.852\n",
      "Accuracy with re-ranking: 0.838\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|███████████▉                                                           | 550/3270 [04:12<23:15,  1.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 550\n",
      "Accuracy without re-ranking: 0.8418181818181818\n",
      "Accuracy with re-ranking: 0.8381818181818181\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█████████████                                                          | 600/3270 [04:32<16:10,  2.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 600\n",
      "Accuracy without re-ranking: 0.8383333333333334\n",
      "Accuracy with re-ranking: 0.8316666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██████████████                                                         | 650/3270 [04:55<18:55,  2.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 650\n",
      "Accuracy without re-ranking: 0.8369230769230769\n",
      "Accuracy with re-ranking: 0.8276923076923077\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|███████████████▏                                                       | 700/3270 [05:17<18:02,  2.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 700\n",
      "Accuracy without re-ranking: 0.8385714285714285\n",
      "Accuracy with re-ranking: 0.8285714285714286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|████████████████▎                                                      | 750/3270 [05:38<17:34,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 750\n",
      "Accuracy without re-ranking: 0.832\n",
      "Accuracy with re-ranking: 0.8266666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|█████████████████▎                                                     | 800/3270 [05:59<17:45,  2.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 800\n",
      "Accuracy without re-ranking: 0.835\n",
      "Accuracy with re-ranking: 0.82875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██████████████████▍                                                    | 850/3270 [06:21<19:12,  2.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 850\n",
      "Accuracy without re-ranking: 0.8352941176470589\n",
      "Accuracy with re-ranking: 0.8282352941176471\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|███████████████████▌                                                   | 900/3270 [06:41<14:34,  2.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 900\n",
      "Accuracy without re-ranking: 0.8344444444444444\n",
      "Accuracy with re-ranking: 0.8266666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|████████████████████▋                                                  | 950/3270 [07:04<20:37,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 950\n",
      "Accuracy without re-ranking: 0.8378947368421052\n",
      "Accuracy with re-ranking: 0.8252631578947368\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|█████████████████████▍                                                | 1000/3270 [07:27<14:24,  2.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1000\n",
      "Accuracy without re-ranking: 0.84\n",
      "Accuracy with re-ranking: 0.826\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|██████████████████████▍                                               | 1050/3270 [07:50<15:45,  2.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1050\n",
      "Accuracy without re-ranking: 0.84\n",
      "Accuracy with re-ranking: 0.8257142857142857\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███████████████████████▌                                              | 1100/3270 [08:14<14:23,  2.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1100\n",
      "Accuracy without re-ranking: 0.8418181818181818\n",
      "Accuracy with re-ranking: 0.8272727272727273\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|████████████████████████▌                                             | 1150/3270 [08:35<21:58,  1.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1150\n",
      "Accuracy without re-ranking: 0.8443478260869566\n",
      "Accuracy with re-ranking: 0.8304347826086956\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|█████████████████████████▋                                            | 1200/3270 [08:56<14:30,  2.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1200\n",
      "Accuracy without re-ranking: 0.8458333333333333\n",
      "Accuracy with re-ranking: 0.8325\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|██████████████████████████▊                                           | 1250/3270 [09:21<14:05,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1250\n",
      "Accuracy without re-ranking: 0.8488\n",
      "Accuracy with re-ranking: 0.8352\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███████████████████████████▊                                          | 1300/3270 [09:46<22:47,  1.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1300\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.8361538461538461\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████████████████████████████▉                                         | 1350/3270 [10:10<13:00,  2.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1350\n",
      "Accuracy without re-ranking: 0.8511111111111112\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|█████████████████████████████▉                                        | 1400/3270 [10:37<28:41,  1.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1400\n",
      "Accuracy without re-ranking: 0.8492857142857143\n",
      "Accuracy with re-ranking: 0.8385714285714285\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|███████████████████████████████                                       | 1450/3270 [11:10<24:11,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1450\n",
      "Accuracy without re-ranking: 0.8475862068965517\n",
      "Accuracy with re-ranking: 0.8344827586206897\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████████████████████████████████                                      | 1500/3270 [11:32<10:30,  2.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1500\n",
      "Accuracy without re-ranking: 0.846\n",
      "Accuracy with re-ranking: 0.8313333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|█████████████████████████████████▏                                    | 1550/3270 [11:58<16:05,  1.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1550\n",
      "Accuracy without re-ranking: 0.8464516129032258\n",
      "Accuracy with re-ranking: 0.832258064516129\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|██████████████████████████████████▎                                   | 1600/3270 [12:27<17:12,  1.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1600\n",
      "Accuracy without re-ranking: 0.845625\n",
      "Accuracy with re-ranking: 0.831875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|███████████████████████████████████▎                                  | 1650/3270 [12:50<13:20,  2.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1650\n",
      "Accuracy without re-ranking: 0.8460606060606061\n",
      "Accuracy with re-ranking: 0.8321212121212122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|████████████████████████████████████▍                                 | 1700/3270 [13:21<16:55,  1.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1700\n",
      "Accuracy without re-ranking: 0.8482352941176471\n",
      "Accuracy with re-ranking: 0.8335294117647059\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████████████████████████████████████▍                                | 1750/3270 [13:51<14:25,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1750\n",
      "Accuracy without re-ranking: 0.848\n",
      "Accuracy with re-ranking: 0.8331428571428572\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|██████████████████████████████████████▌                               | 1800/3270 [14:22<12:56,  1.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1800\n",
      "Accuracy without re-ranking: 0.8483333333333334\n",
      "Accuracy with re-ranking: 0.8355555555555556\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|███████████████████████████████████████▌                              | 1850/3270 [14:51<14:33,  1.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1850\n",
      "Accuracy without re-ranking: 0.8475675675675676\n",
      "Accuracy with re-ranking: 0.8356756756756757\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|████████████████████████████████████████▋                             | 1900/3270 [15:20<10:16,  2.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1900\n",
      "Accuracy without re-ranking: 0.8494736842105263\n",
      "Accuracy with re-ranking: 0.8378947368421052\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|█████████████████████████████████████████▋                            | 1950/3270 [15:51<11:43,  1.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1950\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.8384615384615385\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████████████████████████████████████████▊                           | 2000/3270 [16:20<12:50,  1.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2000\n",
      "Accuracy without re-ranking: 0.851\n",
      "Accuracy with re-ranking: 0.839\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|███████████████████████████████████████████▉                          | 2050/3270 [16:52<11:14,  1.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2050\n",
      "Accuracy without re-ranking: 0.8526829268292683\n",
      "Accuracy with re-ranking: 0.8414634146341463\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|████████████████████████████████████████████▉                         | 2100/3270 [17:18<09:55,  1.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2100\n",
      "Accuracy without re-ranking: 0.8514285714285714\n",
      "Accuracy with re-ranking: 0.8404761904761905\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████████████████████████████████████████████                        | 2150/3270 [17:43<08:29,  2.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2150\n",
      "Accuracy without re-ranking: 0.8502325581395349\n",
      "Accuracy with re-ranking: 0.8418604651162791\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|███████████████████████████████████████████████                       | 2200/3270 [18:12<08:40,  2.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2200\n",
      "Accuracy without re-ranking: 0.8504545454545455\n",
      "Accuracy with re-ranking: 0.8409090909090909\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|████████████████████████████████████████████████▏                     | 2250/3270 [18:36<07:34,  2.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2250\n",
      "Accuracy without re-ranking: 0.8506666666666667\n",
      "Accuracy with re-ranking: 0.8404444444444444\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|█████████████████████████████████████████████████▏                    | 2300/3270 [18:58<06:26,  2.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2300\n",
      "Accuracy without re-ranking: 0.8504347826086956\n",
      "Accuracy with re-ranking: 0.8386956521739131\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|██████████████████████████████████████████████████▎                   | 2350/3270 [19:22<05:50,  2.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2350\n",
      "Accuracy without re-ranking: 0.8506382978723405\n",
      "Accuracy with re-ranking: 0.8395744680851064\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████████████████████████████████████████████████▍                  | 2400/3270 [19:43<05:22,  2.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2400\n",
      "Accuracy without re-ranking: 0.85125\n",
      "Accuracy with re-ranking: 0.8395833333333333\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|████████████████████████████████████████████████████▍                 | 2450/3270 [20:07<05:54,  2.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2450\n",
      "Accuracy without re-ranking: 0.8526530612244898\n",
      "Accuracy with re-ranking: 0.8420408163265306\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|█████████████████████████████████████████████████████▌                | 2500/3270 [20:29<06:34,  1.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2500\n",
      "Accuracy without re-ranking: 0.8524\n",
      "Accuracy with re-ranking: 0.842\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|██████████████████████████████████████████████████████▌               | 2550/3270 [20:57<08:21,  1.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2550\n",
      "Accuracy without re-ranking: 0.8509803921568627\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████████████████████████████████████████████████████▋              | 2600/3270 [21:22<05:57,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2600\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.8396153846153847\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████████████████████████████████████████████████████▋             | 2650/3270 [21:48<04:08,  2.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2650\n",
      "Accuracy without re-ranking: 0.849811320754717\n",
      "Accuracy with re-ranking: 0.8392452830188679\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|█████████████████████████████████████████████████████████▊            | 2700/3270 [22:10<04:05,  2.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2700\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.8396296296296296\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|██████████████████████████████████████████████████████████▊           | 2750/3270 [22:32<05:25,  1.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2750\n",
      "Accuracy without re-ranking: 0.8501818181818181\n",
      "Accuracy with re-ranking: 0.8389090909090909\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|███████████████████████████████████████████████████████████▉          | 2800/3270 [23:01<04:11,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2800\n",
      "Accuracy without re-ranking: 0.8492857142857143\n",
      "Accuracy with re-ranking: 0.8389285714285715\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|█████████████████████████████████████████████████████████████         | 2850/3270 [23:26<03:56,  1.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2850\n",
      "Accuracy without re-ranking: 0.8501754385964913\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|██████████████████████████████████████████████████████████████        | 2900/3270 [23:56<03:23,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2900\n",
      "Accuracy without re-ranking: 0.8510344827586207\n",
      "Accuracy with re-ranking: 0.8396551724137931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|███████████████████████████████████████████████████████████████▏      | 2950/3270 [24:25<02:41,  1.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2950\n",
      "Accuracy without re-ranking: 0.8501694915254238\n",
      "Accuracy with re-ranking: 0.8396610169491525\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|████████████████████████████████████████████████████████████████▏     | 3000/3270 [24:52<02:02,  2.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3000\n",
      "Accuracy without re-ranking: 0.851\n",
      "Accuracy with re-ranking: 0.8413333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████████████████████████████████████████████████████████████▎    | 3050/3270 [25:15<01:22,  2.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3050\n",
      "Accuracy without re-ranking: 0.8511475409836066\n",
      "Accuracy with re-ranking: 0.8422950819672131\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|██████████████████████████████████████████████████████████████████▎   | 3100/3270 [25:38<01:08,  2.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3100\n",
      "Accuracy without re-ranking: 0.8522580645161291\n",
      "Accuracy with re-ranking: 0.8422580645161291\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|███████████████████████████████████████████████████████████████████▍  | 3150/3270 [26:01<00:53,  2.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3150\n",
      "Accuracy without re-ranking: 0.8526984126984127\n",
      "Accuracy with re-ranking: 0.8428571428571429\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|████████████████████████████████████████████████████████████████████▌ | 3200/3270 [26:23<00:23,  2.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3200\n",
      "Accuracy without re-ranking: 0.8525\n",
      "Accuracy with re-ranking: 0.8421875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████████████████████████████████████████████████████████████████▌| 3250/3270 [26:43<00:07,  2.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3250\n",
      "Accuracy without re-ranking: 0.8526153846153847\n",
      "Accuracy with re-ranking: 0.8415384615384616\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████| 3270/3270 [26:53<00:00,  2.03it/s]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "print_every = 50\n",
    "predictions = []\n",
    "for question in tqdm(val_sample['question']):\n",
    "    retrieved_hash, reranked_hash = eval_ranking(question, newer_cross_encoder, top_k=3)\n",
    "    correct_hash = q_to_hash[question]\n",
    "    predictions.append((retrieved_hash == correct_hash, reranked_hash == correct_hash))\n",
    "    i += 1\n",
    "    if i % print_every == 0:\n",
    "        print(f'Step {i}')\n",
    "        raw_accuracy = sum([p[0] for p in predictions])/len(predictions)\n",
    "        reranked_accuracy = sum([p[1] for p in predictions])/len(predictions)\n",
    "\n",
    "        print(f'Accuracy without re-ranking: {raw_accuracy}')\n",
    "        print(f'Accuracy with re-ranking: {reranked_accuracy}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "502d5e33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cross-encoder: <sentence_transformers.cross_encoder.CrossEncoder.CrossEncoder object at 0x158c9cc70>\n",
      "Accuracy without re-ranking: 0.8522935779816514\n",
      "Accuracy with re-ranking: 0.8418960244648318\n"
     ]
    }
   ],
   "source": [
    "raw_accuracy = sum([p[0] for p in predictions])/len(predictions)\n",
    "reranked_accuracy = sum([p[1] for p in predictions])/len(predictions)\n",
    "\n",
    "print(f'Using cross-encoder: {newer_cross_encoder.config._name_or_path}')\n",
    "print(f'Accuracy without re-ranking: {raw_accuracy}')\n",
    "print(f'Accuracy with re-ranking: {reranked_accuracy}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea30153c",
   "metadata": {},
   "source": [
    "# Fine-tuning re-ranker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "5ecc386f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_cross-encoder_scratch.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "51bf1607",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import InputExample, losses, evaluation\n",
    "from torch.utils.data import DataLoader\n",
    "from random import shuffle\n",
    "\n",
    "shuffled_training_passages = dataset['train']['passage'].copy()\n",
    "shuffle(shuffled_training_passages)\n",
    "\n",
    "\n",
    "train_samples = [\n",
    "  InputExample(texts=[d['question'], d['passage']], label=1) for d in dataset['train']\n",
    "]\n",
    "\n",
    "# add some negative example\n",
    "for i in range(1):\n",
    "    train_samples += [\n",
    "      InputExample(texts=[d['question'], shuffled_training_passages[i]], label=0) for i, d in enumerate(dataset['train'])\n",
    "    ]\n",
    "\n",
    "shuffle(train_samples)\n",
    "\n",
    "# running the risk of overfitting on my data but maybe I want that. \n",
    "#  Combined with sufficient input and output validation, we can make a viable product with a model overfit to my data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "56a817c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18854"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "6b58d2d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-12-v2', num_labels=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "8e7eaa40",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'guid': '',\n",
       " 'texts': ['can a battery charger be used to jumpstart a car',\n",
       "  \"Motorists and service garages often have portable battery chargers operated from AC power. Very small ``trickle'' chargers are intended only to maintain a charge on a parked or stored vehicle, but larger chargers can put enough charge into a battery to allow a start within a few minutes. Battery chargers may be strictly manual, or may include controls for time and charging voltage. Battery chargers that apply high voltage (for example, more than 14.4 volts on a 12 volt nominal system) will result in emission of hydrogen gas from the battery, which may damage it or create an explosion risk. A battery may be recharged without removal from the vehicle, although in a typical roadside situation no convenient source of power may be nearby.\"],\n",
       " 'label': 1}"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_samples[0].__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "57e1368d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.79540616"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(train_samples[0].texts, activation_fct=nn.Sigmoid())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "a943dff4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warmup-steps: 95\n"
     ]
    }
   ],
   "source": [
    "from sentence_transformers.cross_encoder.evaluation import CECorrelationEvaluator, CEBinaryClassificationEvaluator\n",
    "import math\n",
    "import torch\n",
    "from random import sample\n",
    "\n",
    "logger.setLevel(logging.DEBUG)  # just to get some logs\n",
    "\n",
    "num_epochs = 2\n",
    "\n",
    "model_save_path = './fine_tuned_ir_cross_encoder'\n",
    "\n",
    "# train_samples = sample(train_samples, 1000)\n",
    "\n",
    "# int(len(train_samples)*.8)\n",
    "train_dataloader = DataLoader(train_samples[:int(len(train_samples)*.8)], shuffle=True, batch_size=32)\n",
    "\n",
    "# An evaluator for training performance\n",
    "evaluator = CEBinaryClassificationEvaluator.from_input_examples(train_samples[-int(len(train_samples)*.8):], name='test')\n",
    "\n",
    "# Rule of thumb for warmup steps\n",
    "warmup_steps = math.ceil(len(train_dataloader) * num_epochs * 0.1)  # 10% of train data for warm-up\n",
    "print(f\"Warmup-steps: {warmup_steps}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05597c3c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "0de4dbc8",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9981043137096746\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b48955070e149269f0450cd4e20b5d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0782b84fa2f042a68bbbd2005b8307f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "beeb917f2b404da797e93cbdb3ceba0e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# # ##### Load model and eval on test set\n",
    "print(evaluator(model))\n",
    "\n",
    "# Train the model\n",
    "model.fit(\n",
    "    train_dataloader=train_dataloader,\n",
    "    loss_fct=losses.nn.CrossEntropyLoss(),\n",
    "    activation_fct=nn.Sigmoid(),\n",
    "    evaluator=evaluator,\n",
    "    epochs=num_epochs,\n",
    "    warmup_steps=warmup_steps,\n",
    "    output_path=model_save_pathm,\n",
    "    use_amp=True\n",
    ")\n",
    "\n",
    "# ##### Load model and eval on test set\n",
    "# print(evaluator(model))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "2f747537",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9998566\n",
      "8.84999\n"
     ]
    }
   ],
   "source": [
    "finetuned = CrossEncoder(model_save_path)\n",
    "\n",
    "print(finetuned.predict(['hello', 'hi'], activation_fct=nn.Sigmoid()))\n",
    "print(finetuned.predict(['hello', 'hi'], activation_fct=nn.Identity()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "392aff2c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "94cf736e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█                                                                       | 50/3270 [00:34<27:59,  1.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 50\n",
      "Accuracy without re-ranking: 0.88\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|██▏                                                                    | 100/3270 [00:57<29:31,  1.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 100\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|███▎                                                                   | 151/3270 [01:23<23:28,  2.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 150\n",
      "Accuracy without re-ranking: 0.86\n",
      "Accuracy with re-ranking: 0.8266666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|████▎                                                                  | 200/3270 [01:48<25:35,  2.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 200\n",
      "Accuracy without re-ranking: 0.865\n",
      "Accuracy with re-ranking: 0.82\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|█████▍                                                                 | 250/3270 [02:10<19:41,  2.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 250\n",
      "Accuracy without re-ranking: 0.872\n",
      "Accuracy with re-ranking: 0.816\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|██████▌                                                                | 300/3270 [02:33<22:06,  2.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 300\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.8133333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|███████▌                                                               | 350/3270 [03:00<23:55,  2.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 350\n",
      "Accuracy without re-ranking: 0.86\n",
      "Accuracy with re-ranking: 0.8285714285714286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|████████▋                                                              | 400/3270 [03:23<24:56,  1.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 400\n",
      "Accuracy without re-ranking: 0.8625\n",
      "Accuracy with re-ranking: 0.8325\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█████████▊                                                             | 450/3270 [03:49<17:03,  2.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 450\n",
      "Accuracy without re-ranking: 0.8577777777777778\n",
      "Accuracy with re-ranking: 0.8311111111111111\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|██████████▊                                                            | 500/3270 [04:12<17:15,  2.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 500\n",
      "Accuracy without re-ranking: 0.852\n",
      "Accuracy with re-ranking: 0.832\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|███████████▉                                                           | 550/3270 [04:33<16:28,  2.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 550\n",
      "Accuracy without re-ranking: 0.8418181818181818\n",
      "Accuracy with re-ranking: 0.8309090909090909\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█████████████                                                          | 600/3270 [04:54<15:37,  2.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 600\n",
      "Accuracy without re-ranking: 0.8383333333333334\n",
      "Accuracy with re-ranking: 0.8266666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██████████████                                                         | 650/3270 [05:20<29:06,  1.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 650\n",
      "Accuracy without re-ranking: 0.8369230769230769\n",
      "Accuracy with re-ranking: 0.823076923076923\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|███████████████▏                                                       | 700/3270 [05:42<21:59,  1.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 700\n",
      "Accuracy without re-ranking: 0.8385714285714285\n",
      "Accuracy with re-ranking: 0.8242857142857143\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|████████████████▎                                                      | 750/3270 [06:06<17:09,  2.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 750\n",
      "Accuracy without re-ranking: 0.832\n",
      "Accuracy with re-ranking: 0.8213333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|█████████████████▎                                                     | 800/3270 [06:32<19:17,  2.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 800\n",
      "Accuracy without re-ranking: 0.835\n",
      "Accuracy with re-ranking: 0.82375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██████████████████▍                                                    | 850/3270 [06:57<19:37,  2.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 850\n",
      "Accuracy without re-ranking: 0.8352941176470589\n",
      "Accuracy with re-ranking: 0.8211764705882353\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|███████████████████▌                                                   | 900/3270 [07:21<20:04,  1.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 900\n",
      "Accuracy without re-ranking: 0.8344444444444444\n",
      "Accuracy with re-ranking: 0.8177777777777778\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|████████████████████▋                                                  | 950/3270 [07:45<13:52,  2.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 950\n",
      "Accuracy without re-ranking: 0.8378947368421052\n",
      "Accuracy with re-ranking: 0.8210526315789474\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|█████████████████████▍                                                | 1000/3270 [08:08<21:13,  1.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1000\n",
      "Accuracy without re-ranking: 0.84\n",
      "Accuracy with re-ranking: 0.823\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|██████████████████████▍                                               | 1050/3270 [08:31<17:24,  2.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1050\n",
      "Accuracy without re-ranking: 0.84\n",
      "Accuracy with re-ranking: 0.8238095238095238\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███████████████████████▌                                              | 1100/3270 [08:57<28:45,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1100\n",
      "Accuracy without re-ranking: 0.8418181818181818\n",
      "Accuracy with re-ranking: 0.8254545454545454\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|████████████████████████▌                                             | 1150/3270 [09:20<12:34,  2.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1150\n",
      "Accuracy without re-ranking: 0.8443478260869566\n",
      "Accuracy with re-ranking: 0.8295652173913044\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|█████████████████████████▋                                            | 1200/3270 [09:50<23:11,  1.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1200\n",
      "Accuracy without re-ranking: 0.8458333333333333\n",
      "Accuracy with re-ranking: 0.8333333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|██████████████████████████▊                                           | 1250/3270 [10:14<17:05,  1.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1250\n",
      "Accuracy without re-ranking: 0.8488\n",
      "Accuracy with re-ranking: 0.836\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███████████████████████████▊                                          | 1300/3270 [10:37<16:20,  2.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1300\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.8376923076923077\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████████████████████████████▉                                         | 1350/3270 [11:00<12:18,  2.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1350\n",
      "Accuracy without re-ranking: 0.8511111111111112\n",
      "Accuracy with re-ranking: 0.8392592592592593\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|█████████████████████████████▉                                        | 1400/3270 [11:31<16:49,  1.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1400\n",
      "Accuracy without re-ranking: 0.8492857142857143\n",
      "Accuracy with re-ranking: 0.8364285714285714\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|███████████████████████████████                                       | 1450/3270 [11:56<13:51,  2.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1450\n",
      "Accuracy without re-ranking: 0.8475862068965517\n",
      "Accuracy with re-ranking: 0.836551724137931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████████████████████████████████                                      | 1500/3270 [12:26<27:23,  1.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1500\n",
      "Accuracy without re-ranking: 0.846\n",
      "Accuracy with re-ranking: 0.8333333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|█████████████████████████████████▏                                    | 1550/3270 [12:59<12:16,  2.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1550\n",
      "Accuracy without re-ranking: 0.8464516129032258\n",
      "Accuracy with re-ranking: 0.8367741935483871\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|██████████████████████████████████▎                                   | 1600/3270 [13:26<12:14,  2.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1600\n",
      "Accuracy without re-ranking: 0.845625\n",
      "Accuracy with re-ranking: 0.835625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|███████████████████████████████████▎                                  | 1650/3270 [13:54<22:36,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1650\n",
      "Accuracy without re-ranking: 0.8460606060606061\n",
      "Accuracy with re-ranking: 0.8363636363636363\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|████████████████████████████████████▍                                 | 1700/3270 [14:22<13:02,  2.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1700\n",
      "Accuracy without re-ranking: 0.8482352941176471\n",
      "Accuracy with re-ranking: 0.8382352941176471\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████████████████████████████████████▍                                | 1750/3270 [14:51<11:45,  2.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1750\n",
      "Accuracy without re-ranking: 0.848\n",
      "Accuracy with re-ranking: 0.8394285714285714\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|██████████████████████████████████████▌                               | 1800/3270 [15:15<13:31,  1.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1800\n",
      "Accuracy without re-ranking: 0.8483333333333334\n",
      "Accuracy with re-ranking: 0.8416666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|███████████████████████████████████████▌                              | 1850/3270 [15:43<11:41,  2.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1850\n",
      "Accuracy without re-ranking: 0.8475675675675676\n",
      "Accuracy with re-ranking: 0.8416216216216216\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|████████████████████████████████████████▋                             | 1900/3270 [16:06<09:07,  2.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1900\n",
      "Accuracy without re-ranking: 0.8494736842105263\n",
      "Accuracy with re-ranking: 0.8436842105263158\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|█████████████████████████████████████████▋                            | 1950/3270 [16:33<09:21,  2.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1950\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.8441025641025641\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████████████████████████████████████████▊                           | 2000/3270 [16:58<09:06,  2.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2000\n",
      "Accuracy without re-ranking: 0.851\n",
      "Accuracy with re-ranking: 0.846\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|███████████████████████████████████████████▉                          | 2050/3270 [17:29<09:37,  2.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2050\n",
      "Accuracy without re-ranking: 0.8526829268292683\n",
      "Accuracy with re-ranking: 0.848780487804878\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|████████████████████████████████████████████▉                         | 2100/3270 [17:59<07:29,  2.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2100\n",
      "Accuracy without re-ranking: 0.8514285714285714\n",
      "Accuracy with re-ranking: 0.8476190476190476\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████████████████████████████████████████████                        | 2150/3270 [18:24<11:03,  1.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2150\n",
      "Accuracy without re-ranking: 0.8502325581395349\n",
      "Accuracy with re-ranking: 0.8483720930232558\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|███████████████████████████████████████████████                       | 2200/3270 [18:55<06:32,  2.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2200\n",
      "Accuracy without re-ranking: 0.8504545454545455\n",
      "Accuracy with re-ranking: 0.8481818181818181\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|████████████████████████████████████████████████▏                     | 2250/3270 [19:22<08:23,  2.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2250\n",
      "Accuracy without re-ranking: 0.8506666666666667\n",
      "Accuracy with re-ranking: 0.8475555555555555\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|█████████████████████████████████████████████████▏                    | 2300/3270 [19:47<06:57,  2.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2300\n",
      "Accuracy without re-ranking: 0.8504347826086956\n",
      "Accuracy with re-ranking: 0.8465217391304348\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|██████████████████████████████████████████████████▎                   | 2350/3270 [20:17<11:49,  1.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2350\n",
      "Accuracy without re-ranking: 0.8506382978723405\n",
      "Accuracy with re-ranking: 0.8472340425531915\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████████████████████████████████████████████████▍                  | 2400/3270 [20:39<04:52,  2.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2400\n",
      "Accuracy without re-ranking: 0.85125\n",
      "Accuracy with re-ranking: 0.8479166666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|████████████████████████████████████████████████████▍                 | 2450/3270 [21:09<07:23,  1.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2450\n",
      "Accuracy without re-ranking: 0.8526530612244898\n",
      "Accuracy with re-ranking: 0.8497959183673469\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|█████████████████████████████████████████████████████▌                | 2500/3270 [21:35<05:00,  2.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2500\n",
      "Accuracy without re-ranking: 0.8524\n",
      "Accuracy with re-ranking: 0.8492\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|██████████████████████████████████████████████████████▌               | 2550/3270 [21:57<05:31,  2.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2550\n",
      "Accuracy without re-ranking: 0.8509803921568627\n",
      "Accuracy with re-ranking: 0.8470588235294118\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████████████████████████████████████████████████████▋              | 2600/3270 [22:24<06:32,  1.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2600\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.8457692307692307\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████████████████████████████████████████████████████▋             | 2650/3270 [22:53<07:52,  1.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2650\n",
      "Accuracy without re-ranking: 0.849811320754717\n",
      "Accuracy with re-ranking: 0.8456603773584905\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|█████████████████████████████████████████████████████████▊            | 2700/3270 [23:16<04:34,  2.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2700\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.8462962962962963\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|██████████████████████████████████████████████████████████▊           | 2750/3270 [23:42<03:28,  2.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2750\n",
      "Accuracy without re-ranking: 0.8501818181818181\n",
      "Accuracy with re-ranking: 0.8458181818181818\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|███████████████████████████████████████████████████████████▉          | 2800/3270 [24:05<03:52,  2.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2800\n",
      "Accuracy without re-ranking: 0.8492857142857143\n",
      "Accuracy with re-ranking: 0.8457142857142858\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|█████████████████████████████████████████████████████████████         | 2850/3270 [24:31<02:52,  2.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2850\n",
      "Accuracy without re-ranking: 0.8501754385964913\n",
      "Accuracy with re-ranking: 0.8470175438596491\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|██████████████████████████████████████████████████████████████        | 2900/3270 [24:51<02:10,  2.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2900\n",
      "Accuracy without re-ranking: 0.8510344827586207\n",
      "Accuracy with re-ranking: 0.8472413793103448\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|███████████████████████████████████████████████████████████████▏      | 2950/3270 [25:14<02:49,  1.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2950\n",
      "Accuracy without re-ranking: 0.8501694915254238\n",
      "Accuracy with re-ranking: 0.847457627118644\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|████████████████████████████████████████████████████████████████▏     | 3000/3270 [25:48<01:59,  2.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3000\n",
      "Accuracy without re-ranking: 0.851\n",
      "Accuracy with re-ranking: 0.8486666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████████████████████████████████████████████████████████████▎    | 3050/3270 [26:12<01:26,  2.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3050\n",
      "Accuracy without re-ranking: 0.8511475409836066\n",
      "Accuracy with re-ranking: 0.8491803278688524\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|██████████████████████████████████████████████████████████████████▎   | 3100/3270 [26:35<01:36,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3100\n",
      "Accuracy without re-ranking: 0.8522580645161291\n",
      "Accuracy with re-ranking: 0.8493548387096774\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|███████████████████████████████████████████████████████████████████▍  | 3150/3270 [27:01<00:51,  2.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3150\n",
      "Accuracy without re-ranking: 0.8526984126984127\n",
      "Accuracy with re-ranking: 0.8501587301587301\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|████████████████████████████████████████████████████████████████████▌ | 3200/3270 [27:50<00:38,  1.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3200\n",
      "Accuracy without re-ranking: 0.8525\n",
      "Accuracy with re-ranking: 0.8496875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████████████████████████████████████████████████████████████████▌| 3250/3270 [28:18<00:08,  2.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3250\n",
      "Accuracy without re-ranking: 0.8526153846153847\n",
      "Accuracy with re-ranking: 0.8492307692307692\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████| 3270/3270 [28:31<00:00,  1.91it/s]\n"
     ]
    }
   ],
   "source": [
    "# Trying our fine-tuned cross encoder\n",
    "logger.setLevel(logging.CRITICAL)  # just to suppress some logs\n",
    "from tqdm import tqdm\n",
    "\n",
    "i = 0\n",
    "print_every = 50\n",
    "predictions = []\n",
    "for question in tqdm(val_sample['question']):\n",
    "    retrieved_hash, reranked_hash = eval_ranking(question, finetuned, top_k=3)\n",
    "    correct_hash = q_to_hash[question]\n",
    "    predictions.append((retrieved_hash == correct_hash, reranked_hash == correct_hash))\n",
    "    i += 1\n",
    "    if i % print_every == 0:\n",
    "        print(f'Step {i}')\n",
    "        raw_accuracy = sum([p[0] for p in predictions])/len(predictions)\n",
    "        reranked_accuracy = sum([p[1] for p in predictions])/len(predictions)\n",
    "\n",
    "        print(f'Accuracy without re-ranking: {raw_accuracy}')\n",
    "        print(f'Accuracy with re-ranking: {reranked_accuracy}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcb0c0db",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Re-ranking got slightly better after 2 epochs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "2b6e0a9e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cross-encoder: <sentence_transformers.cross_encoder.CrossEncoder.CrossEncoder object at 0x158c9cc70>\n",
      "Accuracy without re-ranking: 0.8522935779816514\n",
      "Accuracy with re-ranking: 0.8495412844036697\n"
     ]
    }
   ],
   "source": [
    "raw_accuracy = sum([p[0] for p in predictions])/len(predictions)\n",
    "reranked_accuracy = sum([p[1] for p in predictions])/len(predictions)\n",
    "\n",
    "print(f'Using cross-encoder: {finetuned.config._name_or_path}')\n",
    "print(f'Accuracy without re-ranking: {raw_accuracy}')\n",
    "print(f'Accuracy with re-ranking: {reranked_accuracy}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f100394",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33810714",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "e250d1f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I am a data scientist\t1.00\t1.00\n",
      "I used to be a data scientist\t0.94\t0.96\n",
      "I am an analyst and machine learning engineer\t0.04\t0.92\n",
      "I am a biologist\t0.01\t0.88\n",
      "I love science fields like biology becasue they use data\t0.30\t0.85\n",
      "I am a baker\t0.00\t0.85\n"
     ]
    }
   ],
   "source": [
    "reference = 'I am a data scientist'\n",
    "candidates = ['I am an analyst and machine learning engineer', 'I am a baker',\n",
    "              'I am a biologist', 'I used to be a data scientist',\n",
    "              'I am a data scientist', 'I love science fields like biology becasue they use data']\n",
    "\n",
    "sentence_combinations = [[reference, text] for text in candidates]\n",
    "cosine_scores = cosine_similarity(get_embeddings([reference] + candidates, engine=ENGINE))[0][1:]\n",
    "\n",
    "# Compute the similarity scores for these combinations\n",
    "similarity_scores = cross_encoder.predict(sentence_combinations, activation_fct=nn.Sigmoid())\n",
    "\n",
    "# Sort the scores in decreasing order\n",
    "sim_scores_argsort = reversed(np.argsort(cosine_scores))\n",
    "\n",
    "# Print the scores\n",
    "for idx in sim_scores_argsort:\n",
    "    print(f\"{candidates[idx]}\\t{similarity_scores[idx]:.2f}\\t{cosine_scores[idx]:.2f}\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50d693d0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "399aceee",
   "metadata": {},
   "outputs": [],
   "source": [
    "pinecone.delete_index(INDEX_NAME)  # delete the index"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f14809e",
   "metadata": {},
   "source": [
    "# OPEN SOURCE ALTERNATIVE TO EMBEDDING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "99138055",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(2, 768)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "model = SentenceTransformer('sentence-transformers/multi-qa-mpnet-base-cos-v1')\n",
    "\n",
    "docs = [\"Around 9 Million people live in London\", \"London is known for its financial district\"]\n",
    "\n",
    "doc_emb = model.encode(docs, batch_size=32, show_progress_bar=True)\n",
    "\n",
    "doc_emb.shape#  == ('2, 768')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ebcfb3bc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dff2d3cd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8269d3392294e1fbe1b562b604c4c87",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/103 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Encode query and documents\n",
    "docs = dataset['validation']['passage']\n",
    "doc_emb = model.encode(docs, batch_size=32, show_progress_bar=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8ce65f85",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "were the us and soviet union allies in the cold war\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800000; text-decoration-color: #800000\">╭─────────────────────────────── </span><span style=\"color: #800000; text-decoration-color: #800000; font-weight: bold\">Traceback </span><span style=\"color: #bf7f7f; text-decoration-color: #bf7f7f; font-weight: bold\">(most recent call last)</span><span style=\"color: #800000; text-decoration-color: #800000\"> ────────────────────────────────╮</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/var/folders/y9/9xqbqkg90tnc0cmm0dxt985m0000gn/T/ipykernel_72597/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">4066378576.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">5</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">&lt;module&gt;</span>     <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000; font-style: italic\">[Errno 2] No such file or directory: </span>                                                            <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000; font-style: italic\">'/var/folders/y9/9xqbqkg90tnc0cmm0dxt985m0000gn/T/ipykernel_72597/4066378576.py'</span>                 <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">╰──────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
       "<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-weight: bold\">NameError: </span>name <span style=\"color: #008000; text-decoration-color: #008000\">'get_results_from_pinecone'</span> is not defined\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[2;33m/var/folders/y9/9xqbqkg90tnc0cmm0dxt985m0000gn/T/ipykernel_72597/\u001b[0m\u001b[1;33m4066378576.py\u001b[0m:\u001b[94m5\u001b[0m in \u001b[92m<module>\u001b[0m     \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[3;31m[Errno 2] No such file or directory: \u001b[0m                                                            \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[3;31m'/var/folders/y9/9xqbqkg90tnc0cmm0dxt985m0000gn/T/ipykernel_72597/4066378576.py'\u001b[0m                 \u001b[31m│\u001b[0m\n",
       "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
       "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'get_results_from_pinecone'\u001b[0m is not defined\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from random import sample\n",
    "\n",
    "query = sample(dataset['validation']['question'], 1)[0]\n",
    "print(query)\n",
    "final_results = get_results_from_pinecone(query, top_k=3, re_rank=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5705a07e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "45c71fb0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6090121269226074 The Cold War was a state of geopolitical tension after World War II between powers in the Eastern Bloc (the Soviet Union and its satellite states) and powers in the Western Bloc (the United States, its NATO allies and others). Historians do not fully agree on the dates, but a common timeframe is the period between 1947, the year the Truman Doctrine, a U.S. foreign policy pledging to aid nations threatened by Soviet expansionism, was announced, and either 1989, when communism fell in Eastern Europe, or 1991, when the Soviet Union collapsed. The term ``cold'' is used because there was no large-scale fighting directly between the two sides, but they each supported major regional wars known as proxy wars.\n",
      "0.48948991298675537 At the start of the war on 1 September 1939, the Allies consisted of France, Poland and the United Kingdom, as well as their dependent states, such as British India. Within days they were joined by the independent Dominions of the British Commonwealth: Australia, Canada, New Zealand and South Africa. After the start of the German invasion of North Europe until the Balkan Campaign, the Netherlands, Belgium, Greece, and Yugoslavia joined the Allies. After first having cooperated with Germany in invading Poland whilst remaining neutral in the Allied-Axis conflict, the Soviet Union perforce joined the Allies in June 1941 after being invaded by Germany. The United States provided war materiel and money all along, and officially joined in December 1941 after the Japanese attack on Pearl Harbor. China had already been in a prolonged war with Japan since the Marco Polo Bridge Incident of 1937, but officially joined the Allies in 1941.\n",
      "0.48948991298675537 At the start of the war on 1 September 1939, the Allies consisted of France, Poland and the United Kingdom, as well as their dependent states, such as British India. Within days they were joined by the independent Dominions of the British Commonwealth: Australia, Canada, New Zealand and South Africa. After the start of the German invasion of North Europe until the Balkan Campaign, the Netherlands, Belgium, Greece, and Yugoslavia joined the Allies. After first having cooperated with Germany in invading Poland whilst remaining neutral in the Allied-Axis conflict, the Soviet Union perforce joined the Allies in June 1941 after being invaded by Germany. The United States provided war materiel and money all along, and officially joined in December 1941 after the Japanese attack on Pearl Harbor. China had already been in a prolonged war with Japan since the Marco Polo Bridge Incident of 1937, but officially joined the Allies in 1941.\n"
     ]
    }
   ],
   "source": [
    "query_emb = model.encode(query)\n",
    "\n",
    "#Compute dot score between query and all document embeddings\n",
    "scores = util.dot_score(query_emb, doc_emb)[0].cpu().tolist()\n",
    "\n",
    "#Combine docs & scores\n",
    "doc_score_pairs = list(zip(docs, scores))\n",
    "\n",
    "#Sort by decreasing score\n",
    "doc_score_pairs = sorted(doc_score_pairs, key=lambda x: x[1], reverse=True)\n",
    "\n",
    "#Output passages & scores\n",
    "for doc, score in doc_score_pairs[:3]:\n",
    "    print(score, doc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8533d9cb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "93280be8",
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.setLevel(logging.CRITICAL)  # just to suppress some logs\n",
    "\n",
    "\n",
    "def eval_ranking_open_source(query, cross_encoder, top_k=3):\n",
    "    query_emb = model.encode(query)\n",
    "\n",
    "    #Compute dot score between query and all document embeddings\n",
    "    scores = util.dot_score(query_emb, doc_emb)[0].cpu().tolist()\n",
    "\n",
    "    #Combine docs & scores\n",
    "    doc_score_pairs = list(zip(docs, scores))\n",
    "\n",
    "    #Sort by decreasing score\n",
    "    doc_score_pairs = sorted(doc_score_pairs, key=lambda x: x[1], reverse=True)[:top_k]\n",
    "\n",
    "    retrieved_hash = my_hash(doc_score_pairs[0][0])\n",
    "    if cross_encoder:\n",
    "        sentence_combinations = [[query, doc_score_pair[0]] for doc_score_pair in doc_score_pairs]\n",
    "        similarity_scores = cross_encoder.predict(sentence_combinations)\n",
    "        sim_scores_argsort = list(reversed(np.argsort(similarity_scores)))\n",
    "        reranked_hash = my_hash(doc_score_pairs[sim_scores_argsort[0]][0])\n",
    "    else:\n",
    "        reranked_hash = None\n",
    "    return retrieved_hash, reranked_hash\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "40b4e5aa",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('932d90b691c75f972ccb0182c4be1977', '932d90b691c75f972ccb0182c4be1977')"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_ranking_open_source(query, finetuned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "8faf0c1a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█                                                                       | 50/3270 [00:07<09:44,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 50\n",
      "Accuracy without re-ranking: 0.82\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|██▏                                                                    | 101/3270 [00:15<07:26,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 100\n",
      "Accuracy without re-ranking: 0.83\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|███▎                                                                   | 151/3270 [00:23<07:31,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 150\n",
      "Accuracy without re-ranking: 0.8466666666666667\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|████▎                                                                  | 200/3270 [00:31<11:45,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 200\n",
      "Accuracy without re-ranking: 0.84\n",
      "Accuracy with re-ranking: 0.825\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|█████▍                                                                 | 251/3270 [00:39<08:13,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 250\n",
      "Accuracy without re-ranking: 0.832\n",
      "Accuracy with re-ranking: 0.82\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|██████▌                                                                | 301/3270 [00:47<06:26,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 300\n",
      "Accuracy without re-ranking: 0.8233333333333334\n",
      "Accuracy with re-ranking: 0.8233333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|███████▌                                                               | 351/3270 [00:54<06:28,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 350\n",
      "Accuracy without re-ranking: 0.8428571428571429\n",
      "Accuracy with re-ranking: 0.8371428571428572\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|████████▋                                                              | 401/3270 [01:01<06:08,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 400\n",
      "Accuracy without re-ranking: 0.835\n",
      "Accuracy with re-ranking: 0.8375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█████████▊                                                             | 451/3270 [01:10<09:55,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 450\n",
      "Accuracy without re-ranking: 0.8311111111111111\n",
      "Accuracy with re-ranking: 0.8355555555555556\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|██████████▉                                                            | 501/3270 [01:17<07:53,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 500\n",
      "Accuracy without re-ranking: 0.83\n",
      "Accuracy with re-ranking: 0.834\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|███████████▉                                                           | 550/3270 [01:25<07:16,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 550\n",
      "Accuracy without re-ranking: 0.82\n",
      "Accuracy with re-ranking: 0.8272727272727273\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█████████████                                                          | 601/3270 [01:32<07:05,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 600\n",
      "Accuracy without re-ranking: 0.82\n",
      "Accuracy with re-ranking: 0.8233333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██████████████▏                                                        | 652/3270 [01:41<05:09,  8.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 650\n",
      "Accuracy without re-ranking: 0.816923076923077\n",
      "Accuracy with re-ranking: 0.8184615384615385\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|███████████████▏                                                       | 701/3270 [01:48<06:28,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 700\n",
      "Accuracy without re-ranking: 0.8214285714285714\n",
      "Accuracy with re-ranking: 0.8214285714285714\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|████████████████▎                                                      | 750/3270 [01:55<06:40,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 750\n",
      "Accuracy without re-ranking: 0.816\n",
      "Accuracy with re-ranking: 0.82\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|█████████████████▎                                                     | 800/3270 [02:02<05:14,  7.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 800\n",
      "Accuracy without re-ranking: 0.815\n",
      "Accuracy with re-ranking: 0.81875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██████████████████▍                                                    | 851/3270 [02:10<06:58,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 850\n",
      "Accuracy without re-ranking: 0.8129411764705883\n",
      "Accuracy with re-ranking: 0.8152941176470588\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|███████████████████▌                                                   | 901/3270 [02:19<06:05,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 900\n",
      "Accuracy without re-ranking: 0.8088888888888889\n",
      "Accuracy with re-ranking: 0.8111111111111111\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|████████████████████▋                                                  | 950/3270 [02:28<06:03,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 950\n",
      "Accuracy without re-ranking: 0.8052631578947368\n",
      "Accuracy with re-ranking: 0.8147368421052632\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|█████████████████████▍                                                | 1000/3270 [02:37<08:52,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1000\n",
      "Accuracy without re-ranking: 0.805\n",
      "Accuracy with re-ranking: 0.814\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|██████████████████████▍                                               | 1051/3270 [02:46<05:09,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1050\n",
      "Accuracy without re-ranking: 0.8047619047619048\n",
      "Accuracy with re-ranking: 0.8142857142857143\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███████████████████████▌                                              | 1101/3270 [02:55<06:37,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1100\n",
      "Accuracy without re-ranking: 0.8054545454545454\n",
      "Accuracy with re-ranking: 0.8145454545454546\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|████████████████████████▋                                             | 1151/3270 [03:01<05:02,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1150\n",
      "Accuracy without re-ranking: 0.808695652173913\n",
      "Accuracy with re-ranking: 0.8182608695652174\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|█████████████████████████▋                                            | 1201/3270 [03:10<05:19,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1200\n",
      "Accuracy without re-ranking: 0.8125\n",
      "Accuracy with re-ranking: 0.8225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|██████████████████████████▊                                           | 1251/3270 [03:17<04:52,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1250\n",
      "Accuracy without re-ranking: 0.8144\n",
      "Accuracy with re-ranking: 0.8248\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███████████████████████████▊                                          | 1300/3270 [03:26<06:12,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1300\n",
      "Accuracy without re-ranking: 0.8146153846153846\n",
      "Accuracy with re-ranking: 0.8276923076923077\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████████████████████████████▉                                         | 1351/3270 [03:36<07:20,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1350\n",
      "Accuracy without re-ranking: 0.8148148148148148\n",
      "Accuracy with re-ranking: 0.8288888888888889\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|█████████████████████████████▉                                        | 1401/3270 [03:46<05:30,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1400\n",
      "Accuracy without re-ranking: 0.8128571428571428\n",
      "Accuracy with re-ranking: 0.8271428571428572\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|███████████████████████████████                                       | 1450/3270 [03:55<04:25,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1450\n",
      "Accuracy without re-ranking: 0.8103448275862069\n",
      "Accuracy with re-ranking: 0.8255172413793104\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████████████████████████████████▏                                     | 1501/3270 [04:03<03:52,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1500\n",
      "Accuracy without re-ranking: 0.8106666666666666\n",
      "Accuracy with re-ranking: 0.8233333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|█████████████████████████████████▏                                    | 1550/3270 [04:11<06:51,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1550\n",
      "Accuracy without re-ranking: 0.8116129032258065\n",
      "Accuracy with re-ranking: 0.8264516129032258\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|██████████████████████████████████▎                                   | 1601/3270 [04:20<04:29,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1600\n",
      "Accuracy without re-ranking: 0.81125\n",
      "Accuracy with re-ranking: 0.82625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|███████████████████████████████████▎                                  | 1651/3270 [04:27<03:44,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1650\n",
      "Accuracy without re-ranking: 0.8109090909090909\n",
      "Accuracy with re-ranking: 0.8272727272727273\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|████████████████████████████████████▍                                 | 1700/3270 [04:36<05:31,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1700\n",
      "Accuracy without re-ranking: 0.8123529411764706\n",
      "Accuracy with re-ranking: 0.8282352941176471\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████████████████████████████████████▍                                | 1751/3270 [04:46<03:53,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1750\n",
      "Accuracy without re-ranking: 0.8137142857142857\n",
      "Accuracy with re-ranking: 0.8285714285714286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|██████████████████████████████████████▌                               | 1801/3270 [04:54<04:14,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1800\n",
      "Accuracy without re-ranking: 0.8166666666666667\n",
      "Accuracy with re-ranking: 0.8311111111111111\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|███████████████████████████████████████▌                              | 1851/3270 [05:03<03:25,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1850\n",
      "Accuracy without re-ranking: 0.8178378378378378\n",
      "Accuracy with re-ranking: 0.8324324324324325\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|████████████████████████████████████████▋                             | 1901/3270 [05:10<03:04,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1900\n",
      "Accuracy without re-ranking: 0.8189473684210526\n",
      "Accuracy with re-ranking: 0.8336842105263158\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|█████████████████████████████████████████▊                            | 1951/3270 [05:18<03:04,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1950\n",
      "Accuracy without re-ranking: 0.8194871794871795\n",
      "Accuracy with re-ranking: 0.8343589743589743\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████████████████████████████████████████▊                           | 2001/3270 [05:26<03:32,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2000\n",
      "Accuracy without re-ranking: 0.821\n",
      "Accuracy with re-ranking: 0.836\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|███████████████████████████████████████████▉                          | 2050/3270 [05:33<02:45,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2050\n",
      "Accuracy without re-ranking: 0.8234146341463414\n",
      "Accuracy with re-ranking: 0.8385365853658536\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|████████████████████████████████████████████▉                         | 2101/3270 [05:42<02:40,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2100\n",
      "Accuracy without re-ranking: 0.8233333333333334\n",
      "Accuracy with re-ranking: 0.8371428571428572\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████████████████████████████████████████████                        | 2149/3270 [05:48<02:58,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2150\n",
      "Accuracy without re-ranking: 0.8237209302325581\n",
      "Accuracy with re-ranking: 0.8367441860465116\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|███████████████████████████████████████████████                       | 2200/3270 [05:57<02:36,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2200\n",
      "Accuracy without re-ranking: 0.8236363636363636\n",
      "Accuracy with re-ranking: 0.835\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|████████████████████████████████████████████████▏                     | 2251/3270 [06:04<02:03,  8.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2250\n",
      "Accuracy without re-ranking: 0.8235555555555556\n",
      "Accuracy with re-ranking: 0.8342222222222222\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|█████████████████████████████████████████████████▏                    | 2300/3270 [06:12<02:11,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2300\n",
      "Accuracy without re-ranking: 0.8221739130434783\n",
      "Accuracy with re-ranking: 0.8330434782608696\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|██████████████████████████████████████████████████▎                   | 2351/3270 [06:21<02:40,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2350\n",
      "Accuracy without re-ranking: 0.8238297872340425\n",
      "Accuracy with re-ranking: 0.8344680851063829\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████████████████████████████████████████████████▍                  | 2400/3270 [06:28<01:55,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2400\n",
      "Accuracy without re-ranking: 0.8241666666666667\n",
      "Accuracy with re-ranking: 0.8345833333333333\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|████████████████████████████████████████████████████▍                 | 2450/3270 [06:38<02:32,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2450\n",
      "Accuracy without re-ranking: 0.8248979591836735\n",
      "Accuracy with re-ranking: 0.8363265306122449\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|█████████████████████████████████████████████████████▌                | 2501/3270 [06:46<01:59,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2500\n",
      "Accuracy without re-ranking: 0.8244\n",
      "Accuracy with re-ranking: 0.836\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|██████████████████████████████████████████████████████▌               | 2550/3270 [06:53<01:47,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2550\n",
      "Accuracy without re-ranking: 0.8231372549019608\n",
      "Accuracy with re-ranking: 0.8337254901960784\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████████████████████████████████████████████████████▋              | 2601/3270 [07:01<01:47,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2600\n",
      "Accuracy without re-ranking: 0.823076923076923\n",
      "Accuracy with re-ranking: 0.833076923076923\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████████████████████████████████████████████████████▋             | 2651/3270 [07:09<01:36,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2650\n",
      "Accuracy without re-ranking: 0.8233962264150944\n",
      "Accuracy with re-ranking: 0.8328301886792453\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|█████████████████████████████████████████████████████████▊            | 2701/3270 [07:16<01:39,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2700\n",
      "Accuracy without re-ranking: 0.8244444444444444\n",
      "Accuracy with re-ranking: 0.8340740740740741\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|██████████████████████████████████████████████████████████▉           | 2751/3270 [07:24<01:18,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2750\n",
      "Accuracy without re-ranking: 0.8236363636363636\n",
      "Accuracy with re-ranking: 0.8334545454545454\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|███████████████████████████████████████████████████████████▉          | 2801/3270 [07:32<01:18,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2800\n",
      "Accuracy without re-ranking: 0.8235714285714286\n",
      "Accuracy with re-ranking: 0.8332142857142857\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|█████████████████████████████████████████████████████████████         | 2851/3270 [07:42<01:22,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2850\n",
      "Accuracy without re-ranking: 0.8231578947368421\n",
      "Accuracy with re-ranking: 0.8333333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|██████████████████████████████████████████████████████████████        | 2900/3270 [07:50<00:55,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2900\n",
      "Accuracy without re-ranking: 0.823103448275862\n",
      "Accuracy with re-ranking: 0.8344827586206897\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|███████████████████████████████████████████████████████████████▏      | 2950/3270 [07:59<00:55,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2950\n",
      "Accuracy without re-ranking: 0.8213559322033899\n",
      "Accuracy with re-ranking: 0.8349152542372882\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|████████████████████████████████████████████████████████████████▏     | 3001/3270 [08:06<00:34,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3000\n",
      "Accuracy without re-ranking: 0.822\n",
      "Accuracy with re-ranking: 0.8356666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████████████████████████████████████████████████████████████▎    | 3051/3270 [08:13<00:28,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3050\n",
      "Accuracy without re-ranking: 0.8226229508196722\n",
      "Accuracy with re-ranking: 0.8360655737704918\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|██████████████████████████████████████████████████████████████████▍   | 3101/3270 [08:21<00:22,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3100\n",
      "Accuracy without re-ranking: 0.8229032258064516\n",
      "Accuracy with re-ranking: 0.8367741935483871\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|███████████████████████████████████████████████████████████████████▍  | 3151/3270 [08:28<00:16,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3150\n",
      "Accuracy without re-ranking: 0.8241269841269842\n",
      "Accuracy with re-ranking: 0.8377777777777777\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|████████████████████████████████████████████████████████████████████▌ | 3201/3270 [08:37<00:13,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3200\n",
      "Accuracy without re-ranking: 0.8240625\n",
      "Accuracy with re-ranking: 0.8375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████████████████████████████████████████████████████████████████▌| 3251/3270 [08:45<00:02,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3250\n",
      "Accuracy without re-ranking: 0.8233846153846154\n",
      "Accuracy with re-ranking: 0.8375384615384616\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████| 3270/3270 [08:48<00:00,  6.19it/s]\n"
     ]
    }
   ],
   "source": [
    "logger.setLevel(logging.CRITICAL)\n",
    "\n",
    "i = 0\n",
    "print_every = 50\n",
    "predictions = []\n",
    "for question in tqdm(val_sample['question']):\n",
    "    retrieved_hash, reranked_hash = eval_ranking_open_source(question, finetuned, top_k=3)\n",
    "    correct_hash = q_to_hash[question]\n",
    "    predictions.append((retrieved_hash == correct_hash, reranked_hash == correct_hash))\n",
    "    i += 1\n",
    "    if i % print_every == 0:\n",
    "        print(f'Step {i}')\n",
    "        raw_accuracy = sum([p[0] for p in predictions])/len(predictions)\n",
    "        reranked_accuracy = sum([p[1] for p in predictions])/len(predictions)\n",
    "\n",
    "        print(f'Accuracy without re-ranking: {raw_accuracy}')\n",
    "        print(f'Accuracy with re-ranking: {reranked_accuracy}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "a00ab5ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cross-encoder: cross-encoder/mmarco-mMiniLMv2-L12-H384-v1\n",
      "Accuracy without re-ranking: 0.8238532110091743\n",
      "Accuracy with re-ranking: 0.8376146788990826\n"
     ]
    }
   ],
   "source": [
    "raw_accuracy = sum([p[0] for p in predictions])/len(predictions)\n",
    "reranked_accuracy = sum([p[1] for p in predictions])/len(predictions)\n",
    "\n",
    "print(f'Using cross-encoder: {cross_encoder.config._name_or_path}')\n",
    "print(f'Accuracy without re-ranking: {raw_accuracy}')\n",
    "print(f'Accuracy with re-ranking: {reranked_accuracy}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9001472",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
